diff --git a/web/server/codechecker_server/api/mass_store_run.py b/web/server/codechecker_server/api/mass_store_run.py index 64a319991d..af535597b1 100644 --- a/web/server/codechecker_server/api/mass_store_run.py +++ b/web/server/codechecker_server/api/mass_store_run.py @@ -47,9 +47,8 @@ from ..database.database import DBSession from ..database.run_db_model import \ AnalysisInfo, AnalysisInfoChecker, AnalyzerStatistic, \ - BugPathEvent, BugReportPoint, \ + ReportPathData, \ Checker, \ - ExtendedReportData, \ File, FileContent, \ Report as DBReport, ReportAnnotations, ReviewStatus as ReviewStatusRule, \ Run, RunLock as DBRunLock, RunHistory, \ @@ -61,7 +60,6 @@ from ..session_manager import SessionManager from ..task_executors.abstract_task import AbstractTask, TaskCancelHonoured from ..task_executors.task_manager import TaskManager -from .thrift_enum_helper import report_extended_data_type_str LOG = get_logger('server') @@ -1270,44 +1268,91 @@ def __realise_fake_checkers(self, session): .update({"checker_id": chk_obj.id}, synchronize_session=False) - def __add_report_context(self, session, file_path_to_id): + def __add_report_context( + self, + session: DBSession, + file_path_to_id: Dict[str, int] + ): for db_report, report in self.__added_reports: + path_data = [] + used_file_ids = set() + LOG.debug("Storing bug path positions.") - for idx, path_pos in enumerate(report.bug_path_positions): - session.add(BugReportPoint( - path_pos.range.start_line, path_pos.range.start_col, - path_pos.range.end_line, path_pos.range.end_col, - idx, file_path_to_id[path_pos.file.path], db_report.id)) + for path_pos in report.bug_path_positions: + path_data.append({ + "from": { + "row": path_pos.range.start_line, + "col": path_pos.range.start_col + }, + "to": { + "row": path_pos.range.end_line, + "col": path_pos.range.end_col + }, + "type": "path", + "fid": file_path_to_id[path_pos.file.path] + }) + used_file_ids.add(file_path_to_id[path_pos.file.path]) LOG.debug("Storing bug path events.") - for idx, event in enumerate(report.bug_path_events): - session.add(BugPathEvent( - event.range.start_line, event.range.start_col, - event.range.end_line, event.range.end_col, - idx, event.message, file_path_to_id[event.file.path], - db_report.id)) + for event in report.bug_path_events: + path_data.append({ + "from": { + "row": event.range.start_line, + "col": event.range.start_col + }, + "to": { + "row": event.range.end_line, + "col": event.range.end_col + }, + "type": "event", + "msg": event.message, + "fid": file_path_to_id[event.file.path] + }) + used_file_ids.add(file_path_to_id[event.file.path]) LOG.debug("Storing notes.") for note in report.notes: - data_type = report_extended_data_type_str( - ttypes.ExtendedReportDataType.NOTE) - - session.add(ExtendedReportData( - note.range.start_line, note.range.start_col, - note.range.end_line, note.range.end_col, - note.message, file_path_to_id[note.file.path], - db_report.id, data_type)) + path_data.append({ + "from": { + "row": note.range.start_line, + "col": note.range.start_col + }, + "to": { + "row": note.range.end_line, + "col": note.range.end_col + }, + "type": "note", + "msg": note.message, + "fid": file_path_to_id[note.file.path] + }) + used_file_ids.add(file_path_to_id[note.file.path]) LOG.debug("Storing macro expansions.") for macro in report.macro_expansions: - data_type = report_extended_data_type_str( - ttypes.ExtendedReportDataType.MACRO) - - session.add(ExtendedReportData( - macro.range.start_line, macro.range.start_col, - macro.range.end_line, macro.range.end_col, - macro.message, file_path_to_id[macro.file.path], - db_report.id, data_type)) + path_data.append({ + "from": { + "row": macro.range.start_line, + "col": macro.range.start_col + }, + "to": { + "row": macro.range.end_line, + "col": macro.range.end_col + }, + "type": "macro", + "msg": macro.message, + "fid": file_path_to_id[macro.file.path] + }) + used_file_ids.add(file_path_to_id[macro.file.path]) + + report_path_data = ReportPathData(db_report.id, path_data) + # TODO: Here we query the File objects with session.get() that runs + # an SQL SELECT statement, since these files are not cached by this + # session object. We should investigate whether it's possible to + # provide a session object that has the File objects already, in + # order to save extra query time. + report_path_data.files.extend( + map(lambda fid: session.get(File, fid), used_file_ids)) + session.add(report_path_data) if report.annotations: self.__validate_and_add_report_annotations( diff --git a/web/server/codechecker_server/api/report_server.py b/web/server/codechecker_server/api/report_server.py index a6d126d1a1..a8cc801145 100644 --- a/web/server/codechecker_server/api/report_server.py +++ b/web/server/codechecker_server/api/report_server.py @@ -34,7 +34,7 @@ from codechecker_api.codeCheckerDBAccess_v6 import constants, ttypes from codechecker_api.codeCheckerDBAccess_v6.ttypes import \ AnalysisInfoFilter, AnalysisInfoChecker as API_AnalysisInfoChecker, \ - BlameData, BlameInfo, BugPathPos, \ + BlameData, BlameInfo, \ CheckerCount, CheckerStatusVerificationDetail, Commit, CommitAuthor, \ CommentData, \ DetectionStatus, DiffType, \ @@ -66,12 +66,11 @@ from ..database.run_db_model import \ AnalysisInfo, AnalysisInfoChecker as DB_AnalysisInfoChecker, \ AnalyzerStatistic, \ - BugPathEvent, BugReportPoint, \ CleanupPlan, CleanupPlanReportHash, Checker, Comment, \ - ExtendedReportData, \ File, FileContent, \ - Report, ReportAnnotations, ReportAnalysisInfo, ReviewStatus, \ - Run, RunHistory, RunHistoryAnalysisInfo, RunLock, \ + Report, ReportAnnotations, ReportAnalysisInfo, \ + ReportPathData, ReportPathDataFile, \ + ReviewStatus, Run, RunHistory, RunHistoryAnalysisInfo, RunLock, \ SourceComponent, SourceComponentFile, FilterPreset from .common import exc_to_thrift_reqfail @@ -554,57 +553,43 @@ def get_source_component_file_query( def get_reports_by_bugpath_filter_for_single_origin( - session, + session: DBSession, file_filter_q ) -> Set[int]: """ This function returns a query for report IDs that are fully contained within the files specified by the file_filter_q query.""" - LOG.info("get_reports_by_bugpath_filter file_filter_q: %s", file_filter_q) q_report = session.query(Report.id) \ .join(File, File.id == Report.file_id) \ .filter(file_filter_q) - q_bugpathevent = session.query(BugPathEvent.report_id) \ - .join(File, File.id == BugPathEvent.file_id) \ - .filter(file_filter_q) - - q_bugreportpoint = session.query(BugReportPoint.report_id) \ - .join(File, File.id == BugReportPoint.file_id) \ + q_reportpathdata = session.query(ReportPathData.report_id) \ + .join(ReportPathDataFile, + ReportPathData.report_id == + ReportPathDataFile.c.report_path_data_id) \ + .join(File, File.id == ReportPathDataFile.c.file_id) \ .filter(file_filter_q) - q_extendedreportdata = session.query(ExtendedReportData.report_id) \ - .join(File, File.id == ExtendedReportData.file_id) \ + neg_q_reportpathdata = session.query(ReportPathData.report_id) \ + .join(ReportPathDataFile, + ReportPathData.report_id == + ReportPathDataFile.c.report_path_data_id) \ + .join(File, File.id != ReportPathDataFile.c.file_id) \ .filter(file_filter_q) neg_q_report = session.query(Report.id) \ .join(File, File.id != Report.file_id) \ .filter(file_filter_q) - neg_q_bugpathevent = session.query(BugPathEvent.report_id) \ - .join(File, File.id != BugPathEvent.file_id) \ - .filter(file_filter_q) + return q_report.union(q_reportpathdata) \ + .except_(neg_q_report, neg_q_reportpathdata) - neg_q_bugreportpoint = session.query(BugReportPoint.report_id) \ - .join(File, File.id != BugReportPoint.file_id) \ - .filter(file_filter_q) - neg_q_extendedreportdata = session.query(ExtendedReportData.report_id) \ - .join(File, File.id != ExtendedReportData.file_id) \ - .filter(file_filter_q) - - return q_report.union( - q_bugpathevent, - q_bugreportpoint, - q_extendedreportdata).except_( - neg_q_report, - neg_q_bugpathevent, - neg_q_bugreportpoint, - neg_q_extendedreportdata) - - -def get_reports_by_bugpath_filter(session, file_filter_q) -> Set[int]: +def get_reports_by_bugpath_filter( + session: DBSession, + file_filter_q +) -> Set[int]: """ This function returns a query for report IDs that are related to any file described by the query in the second parameter, either because their bug @@ -615,22 +600,14 @@ def get_reports_by_bugpath_filter(session, file_filter_q) -> Set[int]: .join(File, File.id == Report.file_id) \ .filter(file_filter_q) - q_bugpathevent = session.query(BugPathEvent.report_id) \ - .join(File, File.id == BugPathEvent.file_id) \ - .filter(file_filter_q) - - q_bugreportpoint = session.query(BugReportPoint.report_id) \ - .join(File, File.id == BugReportPoint.file_id) \ - .filter(file_filter_q) - - q_extendedreportdata = session.query(ExtendedReportData.report_id) \ - .join(File, File.id == ExtendedReportData.file_id) \ + q_reportpathdata = session.query(ReportPathData.report_id) \ + .join(ReportPathDataFile, + ReportPathData.report_id == + ReportPathDataFile.c.report_path_data_id) \ + .join(File, File.id == ReportPathDataFile.c.file_id) \ .filter(file_filter_q) - return q_report.union( - q_bugpathevent, - q_extendedreportdata, - q_bugreportpoint) + return q_report.union(q_reportpathdata) def get_reports_by_components(session, @@ -890,52 +867,38 @@ def process_run_filter(session, query, run_filter): return query -def get_report_details(session, report_ids): +def get_report_details( + session: DBSession, + report_ids: List[int] +) -> Dict[int, ReportDetails]: """ Returns report details for the given report ids. """ details = {} - # Get bug path events. - bug_path_events = session.query(BugPathEvent, File.filepath) \ - .filter(BugPathEvent.report_id.in_(report_ids)) \ - .outerjoin(File, - File.id == BugPathEvent.file_id) \ - .order_by(BugPathEvent.report_id, BugPathEvent.order) - bug_events_list = defaultdict(list) - for event, file_path in bug_path_events: - report_id = event.report_id - event = bugpathevent_db_to_api(event) - event.filePath = file_path - bug_events_list[report_id].append(event) - - # Get bug report points. - bug_report_points = session.query(BugReportPoint, File.filepath) \ - .filter(BugReportPoint.report_id.in_(report_ids)) \ - .outerjoin(File, - File.id == BugReportPoint.file_id) \ - .order_by(BugReportPoint.report_id, BugReportPoint.order) - bug_point_list = defaultdict(list) - for bug_point, file_path in bug_report_points: - report_id = bug_point.report_id - bug_point = bugreportpoint_db_to_api(bug_point) - bug_point.filePath = file_path - bug_point_list[report_id].append(bug_point) - - # Get extended report data. extended_data_list = defaultdict(list) - q = session.query(ExtendedReportData, File.filepath) \ - .filter(ExtendedReportData.report_id.in_(report_ids)) \ - .outerjoin(File, - File.id == ExtendedReportData.file_id) - for data, file_path in q: - report_id = data.report_id - extended_data = extended_data_db_to_api(data) - extended_data.filePath = file_path - extended_data_list[report_id].append(extended_data) + report_path_data = session.query(ReportPathData) \ + .filter(ReportPathData.report_id.in_(report_ids)) + + for rpd in report_path_data: + files = {f.id: f.filepath for f in rpd.files} + + for pd in rpd.path_data: + if pd["type"] == "event": + event = bugpathevent_db_to_api(pd) + event.filePath = files[pd["fid"]] + bug_events_list[rpd.report_id].append(event) + elif pd["type"] == "path": + bug_point = bugreportpoint_db_to_api(pd) + bug_point.filePath = files[pd["fid"]] + bug_point_list[rpd.report_id].append(bug_point) + else: + extended_data = extended_data_db_to_api(pd) + extended_data.filePath = files[pd["fid"]] + extended_data_list[rpd.report_id].append(extended_data) # Get Comments for report data comment_data_list = defaultdict(list) @@ -958,34 +921,34 @@ def get_report_details(session, report_ids): return details -def bugpathevent_db_to_api(bpe): +def bugpathevent_db_to_api(bpe: Dict) -> ttypes.BugPathEvent: return ttypes.BugPathEvent( - startLine=bpe.line_begin, - startCol=bpe.col_begin, - endLine=bpe.line_end, - endCol=bpe.col_end, - msg=bpe.msg, - fileId=bpe.file_id) + startLine=bpe["from"]["row"], + startCol=bpe["from"]["col"], + endLine=bpe["to"]["row"], + endCol=bpe["to"]["col"], + msg=bpe["msg"], + fileId=bpe["fid"]) -def bugreportpoint_db_to_api(brp): - return BugPathPos( - startLine=brp.line_begin, - startCol=brp.col_begin, - endLine=brp.line_end, - endCol=brp.col_end, - fileId=brp.file_id) +def bugreportpoint_db_to_api(brp: Dict) -> ttypes.BugPathPos: + return ttypes.BugPathPos( + startLine=brp["from"]["row"], + startCol=brp["from"]["col"], + endLine=brp["to"]["row"], + endCol=brp["to"]["col"], + fileId=brp["fid"]) -def extended_data_db_to_api(erd): +def extended_data_db_to_api(erd: Dict) -> ttypes.ExtendedReportData: return ttypes.ExtendedReportData( - type=report_extended_data_type_enum(erd.type), - startLine=erd.line_begin, - startCol=erd.col_begin, - endLine=erd.line_end, - endCol=erd.col_end, - message=erd.message, - fileId=erd.file_id) + type=report_extended_data_type_enum(erd["type"]), + startLine=erd["from"]["row"], + startCol=erd["from"]["col"], + endLine=erd["to"]["row"], + endCol=erd["to"]["col"], + message=erd["msg"], + fileId=erd["fid"]) def comment_data_db_to_api(comm): diff --git a/web/server/codechecker_server/database/db_cleanup.py b/web/server/codechecker_server/database/db_cleanup.py index e1c609aa0b..55d0c8e501 100644 --- a/web/server/codechecker_server/database/db_cleanup.py +++ b/web/server/codechecker_server/database/db_cleanup.py @@ -22,10 +22,10 @@ from .database import DBSession from .run_db_model import \ AnalysisInfo, \ - BugPathEvent, BugReportPoint, \ Comment, Checker, \ File, FileContent, \ - Report, ReportAnalysisInfo, RunHistoryAnalysisInfo, RunLock + Report, ReportAnalysisInfo, ReportPathDataFile, RunHistoryAnalysisInfo, \ + RunLock LOG = get_logger('server') RUN_LOCK_TIMEOUT_IN_DATABASE = 30 * 60 # 30 minutes. @@ -88,19 +88,16 @@ def remove_unused_files(product): LOG.debug("[%s] Garbage collection of dangling files started...", product.endpoint) try: - bpe_files = session.query(BugPathEvent.file_id) \ - .group_by(BugPathEvent.file_id) - brp_files = session.query(BugReportPoint.file_id) \ - .group_by(BugReportPoint.file_id) + files = session.query(ReportPathDataFile.c.file_id) \ + .group_by(ReportPathDataFile.c.file_id) files_to_delete = session.query(File.id) \ - .filter(File.id.notin_(bpe_files), File.id.notin_(brp_files)) + .filter(File.id.notin_(files)) files_to_delete = map(lambda x: x[0], files_to_delete) total_count = 0 for chunk in util.chunks(iter(files_to_delete), chunk_size): - q = session.query(File) \ - .filter(File.id.in_(chunk)) + q = session.query(File).filter(File.id.in_(chunk)) count = q.delete(synchronize_session=False) if count: total_count += count diff --git a/web/server/codechecker_server/database/run_db_model.py b/web/server/codechecker_server/database/run_db_model.py index f240ae7826..d96c6a7700 100644 --- a/web/server/codechecker_server/database/run_db_model.py +++ b/web/server/codechecker_server/database/run_db_model.py @@ -9,13 +9,16 @@ SQLAlchemy ORM model for the analysis run storage database. """ from datetime import datetime, timedelta +import json from math import ceil import os from typing import Optional +import zlib from sqlalchemy import Boolean, Column, DateTime, Enum, ForeignKey, Integer, \ LargeBinary, MetaData, String, UniqueConstraint, Table, Text, JSON from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import relationship from sqlalchemy.sql.expression import true, false @@ -263,106 +266,53 @@ def __init__(self, filepath, content_hash, remote_url, tracking_branch): self.tracking_branch = tracking_branch -class BugPathEvent(Base): - __tablename__ = 'bug_path_events' - - line_begin = Column(Integer) - col_begin = Column(Integer) - line_end = Column(Integer) - col_end = Column(Integer) - - order = Column(Integer, primary_key=True) - - msg = Column(String) - file_id = Column(Integer, ForeignKey('files.id', deferrable=True, - initially="DEFERRED", - ondelete='CASCADE'), index=True) - report_id = Column(Integer, ForeignKey('reports.id', deferrable=True, - initially="DEFERRED", - ondelete='CASCADE'), - index=True, - primary_key=True) - - def __init__(self, line_begin, col_begin, line_end, col_end, - order, msg, file_id, report_id): - self.line_begin, self.col_begin, self.line_end, self.col_end = \ - line_begin, col_begin, line_end, col_end - - self.order = order - self.msg = msg - self.file_id = file_id - self.report_id = report_id - - -class BugReportPoint(Base): - __tablename__ = 'bug_report_points' - - line_begin = Column(Integer) - col_begin = Column(Integer) - line_end = Column(Integer) - col_end = Column(Integer) - - order = Column(Integer, primary_key=True) - - file_id = Column(Integer, ForeignKey('files.id', deferrable=True, - initially="DEFERRED", - ondelete='CASCADE'), index=True) - report_id = Column(Integer, ForeignKey('reports.id', deferrable=True, - initially="DEFERRED", - ondelete='CASCADE'), - index=True, - primary_key=True) - - def __init__(self, line_begin, col_begin, line_end, col_end, - order, file_id, report_id): - self.line_begin, self.col_begin, self.line_end, self.col_end = \ - line_begin, col_begin, line_end, col_end - - self.order = order - self.file_id = file_id - self.report_id = report_id +ReportPathDataFile = Table( + 'report_path_data_files', + Base.metadata, + Column( + 'report_path_data_id', + Integer, + ForeignKey('report_path_data.report_id', + deferrable=True, + initially="DEFERRED", + ondelete="CASCADE"), + index=True), + Column('file_id', Integer, ForeignKey('files.id')) +) -class ExtendedReportData(Base): +class ReportPathData(Base): """ - Store extra information which can help to understand or fix a report. + Store bug path positions, bug events, macro expansions, fix-its, etc. """ - __tablename__ = 'extended_report_data' - - id = Column(Integer, autoincrement=True, primary_key=True) - - report_id = Column(Integer, ForeignKey('reports.id', deferrable=True, - initially="DEFERRED", - ondelete='CASCADE'), - index=True) + __tablename__ = 'report_path_data' - file_id = Column(Integer, ForeignKey('files.id', deferrable=True, - initially="DEFERRED", - ondelete='CASCADE'), index=True) + report_id = Column( + Integer, + ForeignKey( + 'reports.id', + deferrable=True, + initially="DEFERRED", + ondelete='CASCADE'), + primary_key=True) - type = Column(Enum('note', - 'macro', - 'fixit', - name='extended_data_type')) + _path_data = Column("path_data", LargeBinary) - line_begin = Column(Integer) - col_begin = Column(Integer) - line_end = Column(Integer) - col_end = Column(Integer) + @hybrid_property + def path_data(self): + return json.loads(zlib.decompress(self._path_data).decode("utf-8")) - message = Column(String) + @path_data.setter + def path_data(self, data): + self._path_data = zlib.compress( + json.dumps(data).encode("utf-8"), + zlib.Z_BEST_COMPRESSION) - def __init__(self, line_begin, col_begin, line_end, col_end, - message, file_id, report_id, data_type): + files = relationship("File", secondary=ReportPathDataFile) - self.line_begin = line_begin - self.col_begin = col_begin - self.line_end = line_end - self.col_end = col_end - self.message = message - self.file_id = file_id + def __init__(self, report_id, path_data): self.report_id = report_id - self.type = data_type + self.path_data = path_data ReportAnalysisInfo = Table( diff --git a/web/server/codechecker_server/migrations/report/versions/ffb641e13b92_report_path_data_table.py b/web/server/codechecker_server/migrations/report/versions/ffb641e13b92_report_path_data_table.py new file mode 100644 index 0000000000..92d7483e8c --- /dev/null +++ b/web/server/codechecker_server/migrations/report/versions/ffb641e13b92_report_path_data_table.py @@ -0,0 +1,329 @@ +""" +Report path data table + +Revision ID: ffb641e13b92 +Revises: 24c9660f82b1 +Create Date: 2026-06-16 02:50:24.445500 +""" + +from logging import getLogger + +from alembic import op +import sqlalchemy as sa + +import json +import zlib +from collections import defaultdict + +from codechecker_common import util + + +# Revision identifiers, used by Alembic. +revision = 'ffb641e13b92' +down_revision = '24c9660f82b1' +branch_labels = None +depends_on = None + + +def upgrade(): + LOG = getLogger("migration/report") + # ### commands auto generated by Alembic - please adjust! ### + report_path_data_table = op.create_table( + 'report_path_data', + sa.Column('report_id', sa.Integer(), nullable=False), + sa.Column('path_data', sa.LargeBinary(), nullable=True), + sa.ForeignKeyConstraint( + ['report_id'], + ['reports.id'], + name=op.f('fk_report_path_data_report_id_reports'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.PrimaryKeyConstraint('report_id', name=op.f('pk_report_path_data')) + ) + report_path_data_files_table = op.create_table( + 'report_path_data_files', + sa.Column('report_path_data_id', sa.Integer(), nullable=True), + sa.Column('file_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ['file_id'], + ['files.id'], + name=op.f('fk_report_path_data_files_file_id_files')), + sa.ForeignKeyConstraint( + ['report_path_data_id'], + ['report_path_data.report_id'], + name=op.f( + 'fk_report_path_data_files_report_path_data_id_' + 'report_path_data'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True) + ) + + conn = op.get_bind() + + report_ids = conn.execute(sa.text("""SELECT id FROM reports""")) + + for report_id_chunk in util.chunks(report_ids, 100_000): + report_id_chunk = list(map(lambda x: x[0], report_id_chunk)) + + conn.execute(sa.text("BEGIN")) + + report_path_data = defaultdict(lambda: { + 'path_data': [], + 'used_file_ids': set() + }) + + bug_report_points = conn.execute(sa.text(f""" + SELECT * FROM bug_report_points + WHERE report_id IN ({','.join(map(str, report_id_chunk))}) + ORDER BY report_id, "order" + """)) + + for brp in bug_report_points: + report_path_data[brp.report_id]['path_data'].append({ + "from": { + "row": brp.line_begin, + "col": brp.col_begin + }, + "to": { + "row": brp.line_end, + "col": brp.col_end + }, + "type": "path", + "fid": brp.file_id + }) + report_path_data[brp.report_id]['used_file_ids'].add(brp.file_id) + + bug_path_events = conn.execute(sa.text(f""" + SELECT * FROM bug_path_events + WHERE report_id IN ({','.join(map(str, report_id_chunk))}) + ORDER BY report_id, "order" + """)) + + for bpe in bug_path_events: + report_path_data[bpe.report_id]['path_data'].append({ + "from": { + "row": bpe.line_begin, + "col": bpe.col_begin + }, + "to": { + "row": bpe.line_end, + "col": bpe.col_end + }, + "type": "event", + "msg": bpe.msg, + "fid": bpe.file_id + }) + report_path_data[bpe.report_id]['used_file_ids'].add(bpe.file_id) + + extended_report_data = conn.execute(sa.text(f""" + SELECT * FROM extended_report_data + WHERE report_id in ({','.join(map(str, report_id_chunk))}) + """)) + + for erd in extended_report_data: + report_path_data[erd.report_id]['path_data'].append({ + "from": { + "row": erd.line_begin, + "col": erd.col_begin + }, + "to": { + "row": erd.line_end, + "col": erd.col_end + }, + "type": erd.type, + "msg": erd.message, + "fid": erd.file_id + }) + report_path_data[erd.report_id]['used_file_ids'].add(erd.file_id) + + report_data_content = [] + report_data_file_content = [] + + for report_id, data in report_path_data.items(): + compressed_data = zlib.compress( + json.dumps(data['path_data']).encode('utf-8'), + zlib.Z_BEST_COMPRESSION) + + report_data_content.append({ + 'report_id': report_id, + 'path_data': compressed_data + }) + + report_data_file_content.extend([{ + 'report_path_data_id': report_id, + 'file_id': fid + } for fid in data['used_file_ids']]) + + conn.execute( + sa.insert(report_path_data_table), + report_data_content) + + conn.execute( + sa.insert(report_path_data_files_table), + report_data_file_content) + + conn.execute(sa.text("COMMIT")) + + op.create_index( + op.f('ix_report_path_data_files_report_path_data_id'), + 'report_path_data_files', + ['report_path_data_id'], + unique=False) + op.drop_index( + op.f('ix_extended_report_data_file_id'), + table_name='extended_report_data') + op.drop_index( + op.f('ix_extended_report_data_report_id'), + table_name='extended_report_data') + op.drop_table('extended_report_data') + op.drop_index( + op.f('ix_bug_report_points_file_id'), + table_name='bug_report_points') + op.drop_index( + op.f('ix_bug_report_points_report_id'), + table_name='bug_report_points') + op.drop_table('bug_report_points') + op.drop_index( + op.f('ix_bug_path_events_file_id'), + table_name='bug_path_events') + op.drop_index( + op.f('ix_bug_path_events_report_id'), + table_name='bug_path_events') + op.drop_table('bug_path_events') + + # ### end Alembic commands ### + + +def downgrade(): + # TODO: Currently we're not supporting the downgrade of DB schema version. + # This is a script generated by Alembic, but the old tables are not + # populated by data from the new schema. + LOG = getLogger("migration/report") + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + 'bug_path_events', + sa.Column('line_begin', sa.INTEGER(), nullable=True), + sa.Column('col_begin', sa.INTEGER(), nullable=True), + sa.Column('line_end', sa.INTEGER(), nullable=True), + sa.Column('col_end', sa.INTEGER(), nullable=True), + sa.Column('order', sa.INTEGER(), nullable=False), + sa.Column('msg', sa.VARCHAR(), nullable=True), + sa.Column('file_id', sa.INTEGER(), nullable=True), + sa.Column('report_id', sa.INTEGER(), nullable=False), + sa.ForeignKeyConstraint( + ['file_id'], + ['files.id'], + name=op.f('fk_bug_path_events_file_id_files'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.ForeignKeyConstraint( + ['report_id'], + ['reports.id'], + name=op.f('fk_bug_path_events_report_id_reports'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.PrimaryKeyConstraint( + 'order', + 'report_id', + name=op.f('pk_bug_path_events')) + ) + op.create_index( + op.f('ix_bug_path_events_report_id'), + 'bug_path_events', + ['report_id'], + unique=False) + op.create_index( + op.f('ix_bug_path_events_file_id'), + 'bug_path_events', + ['file_id'], + unique=False) + op.create_table( + 'bug_report_points', + sa.Column('line_begin', sa.INTEGER(), nullable=True), + sa.Column('col_begin', sa.INTEGER(), nullable=True), + sa.Column('line_end', sa.INTEGER(), nullable=True), + sa.Column('col_end', sa.INTEGER(), nullable=True), + sa.Column('order', sa.INTEGER(), nullable=False), + sa.Column('file_id', sa.INTEGER(), nullable=True), + sa.Column('report_id', sa.INTEGER(), nullable=False), + sa.ForeignKeyConstraint( + ['file_id'], + ['files.id'], + name=op.f('fk_bug_report_points_file_id_files'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.ForeignKeyConstraint( + ['report_id'], + ['reports.id'], + name=op.f('fk_bug_report_points_report_id_reports'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.PrimaryKeyConstraint( + 'order', + 'report_id', + name=op.f('pk_bug_report_points')) + ) + op.create_index( + op.f('ix_bug_report_points_report_id'), + 'bug_report_points', + ['report_id'], + unique=False) + op.create_index( + op.f('ix_bug_report_points_file_id'), + 'bug_report_points', + ['file_id'], + unique=False) + op.create_table( + 'extended_report_data', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('report_id', sa.INTEGER(), nullable=False), + sa.Column('file_id', sa.INTEGER(), nullable=True), + sa.Column('type', sa.VARCHAR(length=5), nullable=True), + sa.Column('line_begin', sa.INTEGER(), nullable=True), + sa.Column('col_begin', sa.INTEGER(), nullable=True), + sa.Column('line_end', sa.INTEGER(), nullable=True), + sa.Column('col_end', sa.INTEGER(), nullable=True), + sa.Column('message', sa.VARCHAR(), nullable=True), + sa.ForeignKeyConstraint( + ['file_id'], + ['files.id'], + name=op.f('fk_extended_report_data_file_id_files'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.ForeignKeyConstraint( + ['report_id'], + ['reports.id'], + name=op.f('fk_extended_report_data_report_id_reports'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.PrimaryKeyConstraint( + 'id', + name=op.f('pk_extended_report_data')) + ) + op.create_index( + op.f('ix_extended_report_data_report_id'), + 'extended_report_data', + ['report_id'], + unique=False) + op.create_index( + op.f('ix_extended_report_data_file_id'), + 'extended_report_data', + ['file_id'], + unique=False) + op.drop_index( + op.f('ix_report_path_data_files_report_path_data_id'), + table_name='report_path_data_files') + op.drop_table('report_path_data_files') + op.drop_index( + op.f('ix_report_path_data_report_id'), + table_name='report_path_data') + op.drop_table('report_path_data') + # ### end Alembic commands ###