From 7e394376c7e138e05fe15090404f9df260a01c33 Mon Sep 17 00:00:00 2001 From: bruntib Date: Tue, 16 Jun 2026 13:23:25 +0200 Subject: [PATCH] [NFC] Merge report path tables in the database There are 3 database tables for storing information about the report paths: - BugPathEvent - BugReportPoint - ExtendedReportData These are now merged to ReportPathData. These tabels were the biggest ones in terms of database size and their handling and querying was problematic. The new table contains all report path related information as a zip blob. Keeping track of the files touched by the report path is still important, so a join-table is storing this information. --- .../codechecker_server/api/mass_store_run.py | 107 ++++-- .../codechecker_server/api/report_server.py | 181 ++++------ .../codechecker_server/database/db_cleanup.py | 15 +- .../database/run_db_model.py | 128 +++---- .../ffb641e13b92_report_path_data_table.py | 329 ++++++++++++++++++ 5 files changed, 522 insertions(+), 238 deletions(-) create mode 100644 web/server/codechecker_server/migrations/report/versions/ffb641e13b92_report_path_data_table.py diff --git a/web/server/codechecker_server/api/mass_store_run.py b/web/server/codechecker_server/api/mass_store_run.py index 64a319991d..af535597b1 100644 --- a/web/server/codechecker_server/api/mass_store_run.py +++ b/web/server/codechecker_server/api/mass_store_run.py @@ -47,9 +47,8 @@ from ..database.database import DBSession from ..database.run_db_model import \ AnalysisInfo, AnalysisInfoChecker, AnalyzerStatistic, \ - BugPathEvent, BugReportPoint, \ + ReportPathData, \ Checker, \ - ExtendedReportData, \ File, FileContent, \ Report as DBReport, ReportAnnotations, ReviewStatus as ReviewStatusRule, \ Run, RunLock as DBRunLock, RunHistory, \ @@ -61,7 +60,6 @@ from ..session_manager import SessionManager from ..task_executors.abstract_task import AbstractTask, TaskCancelHonoured from ..task_executors.task_manager import TaskManager -from .thrift_enum_helper import report_extended_data_type_str LOG = get_logger('server') @@ -1270,44 +1268,91 @@ def __realise_fake_checkers(self, session): .update({"checker_id": chk_obj.id}, synchronize_session=False) - def __add_report_context(self, session, file_path_to_id): + def __add_report_context( + self, + session: DBSession, + file_path_to_id: Dict[str, int] + ): for db_report, report in self.__added_reports: + path_data = [] + used_file_ids = set() + LOG.debug("Storing bug path positions.") - for idx, path_pos in enumerate(report.bug_path_positions): - session.add(BugReportPoint( - path_pos.range.start_line, path_pos.range.start_col, - path_pos.range.end_line, path_pos.range.end_col, - idx, file_path_to_id[path_pos.file.path], db_report.id)) + for path_pos in report.bug_path_positions: + path_data.append({ + "from": { + "row": path_pos.range.start_line, + "col": path_pos.range.start_col + }, + "to": { + "row": path_pos.range.end_line, + "col": path_pos.range.end_col + }, + "type": "path", + "fid": file_path_to_id[path_pos.file.path] + }) + used_file_ids.add(file_path_to_id[path_pos.file.path]) LOG.debug("Storing bug path events.") - for idx, event in enumerate(report.bug_path_events): - session.add(BugPathEvent( - event.range.start_line, event.range.start_col, - event.range.end_line, event.range.end_col, - idx, event.message, file_path_to_id[event.file.path], - db_report.id)) + for event in report.bug_path_events: + path_data.append({ + "from": { + "row": event.range.start_line, + "col": event.range.start_col + }, + "to": { + "row": event.range.end_line, + "col": event.range.end_col + }, + "type": "event", + "msg": event.message, + "fid": file_path_to_id[event.file.path] + }) + used_file_ids.add(file_path_to_id[event.file.path]) LOG.debug("Storing notes.") for note in report.notes: - data_type = report_extended_data_type_str( - ttypes.ExtendedReportDataType.NOTE) - - session.add(ExtendedReportData( - note.range.start_line, note.range.start_col, - note.range.end_line, note.range.end_col, - note.message, file_path_to_id[note.file.path], - db_report.id, data_type)) + path_data.append({ + "from": { + "row": note.range.start_line, + "col": note.range.start_col + }, + "to": { + "row": note.range.end_line, + "col": note.range.end_col + }, + "type": "note", + "msg": note.message, + "fid": file_path_to_id[note.file.path] + }) + used_file_ids.add(file_path_to_id[note.file.path]) LOG.debug("Storing macro expansions.") for macro in report.macro_expansions: - data_type = report_extended_data_type_str( - ttypes.ExtendedReportDataType.MACRO) - - session.add(ExtendedReportData( - macro.range.start_line, macro.range.start_col, - macro.range.end_line, macro.range.end_col, - macro.message, file_path_to_id[macro.file.path], - db_report.id, data_type)) + path_data.append({ + "from": { + "row": macro.range.start_line, + "col": macro.range.start_col + }, + "to": { + "row": macro.range.end_line, + "col": macro.range.end_col + }, + "type": "macro", + "msg": macro.message, + "fid": file_path_to_id[macro.file.path] + }) + used_file_ids.add(file_path_to_id[macro.file.path]) + + report_path_data = ReportPathData(db_report.id, path_data) + # TODO: Here we query the File objects with session.get() that runs + # an SQL SELECT statement, since these files are not cached by this + # session object. We should investigate whether it's possible to + # provide a session object that has the File objects already, in + # order to save extra query time. + report_path_data.files.extend( + map(lambda fid: session.get(File, fid), used_file_ids)) + session.add(report_path_data) if report.annotations: self.__validate_and_add_report_annotations( diff --git a/web/server/codechecker_server/api/report_server.py b/web/server/codechecker_server/api/report_server.py index a6d126d1a1..a8cc801145 100644 --- a/web/server/codechecker_server/api/report_server.py +++ b/web/server/codechecker_server/api/report_server.py @@ -34,7 +34,7 @@ from codechecker_api.codeCheckerDBAccess_v6 import constants, ttypes from codechecker_api.codeCheckerDBAccess_v6.ttypes import \ AnalysisInfoFilter, AnalysisInfoChecker as API_AnalysisInfoChecker, \ - BlameData, BlameInfo, BugPathPos, \ + BlameData, BlameInfo, \ CheckerCount, CheckerStatusVerificationDetail, Commit, CommitAuthor, \ CommentData, \ DetectionStatus, DiffType, \ @@ -66,12 +66,11 @@ from ..database.run_db_model import \ AnalysisInfo, AnalysisInfoChecker as DB_AnalysisInfoChecker, \ AnalyzerStatistic, \ - BugPathEvent, BugReportPoint, \ CleanupPlan, CleanupPlanReportHash, Checker, Comment, \ - ExtendedReportData, \ File, FileContent, \ - Report, ReportAnnotations, ReportAnalysisInfo, ReviewStatus, \ - Run, RunHistory, RunHistoryAnalysisInfo, RunLock, \ + Report, ReportAnnotations, ReportAnalysisInfo, \ + ReportPathData, ReportPathDataFile, \ + ReviewStatus, Run, RunHistory, RunHistoryAnalysisInfo, RunLock, \ SourceComponent, SourceComponentFile, FilterPreset from .common import exc_to_thrift_reqfail @@ -554,57 +553,43 @@ def get_source_component_file_query( def get_reports_by_bugpath_filter_for_single_origin( - session, + session: DBSession, file_filter_q ) -> Set[int]: """ This function returns a query for report IDs that are fully contained within the files specified by the file_filter_q query.""" - LOG.info("get_reports_by_bugpath_filter file_filter_q: %s", file_filter_q) q_report = session.query(Report.id) \ .join(File, File.id == Report.file_id) \ .filter(file_filter_q) - q_bugpathevent = session.query(BugPathEvent.report_id) \ - .join(File, File.id == BugPathEvent.file_id) \ - .filter(file_filter_q) - - q_bugreportpoint = session.query(BugReportPoint.report_id) \ - .join(File, File.id == BugReportPoint.file_id) \ + q_reportpathdata = session.query(ReportPathData.report_id) \ + .join(ReportPathDataFile, + ReportPathData.report_id == + ReportPathDataFile.c.report_path_data_id) \ + .join(File, File.id == ReportPathDataFile.c.file_id) \ .filter(file_filter_q) - q_extendedreportdata = session.query(ExtendedReportData.report_id) \ - .join(File, File.id == ExtendedReportData.file_id) \ + neg_q_reportpathdata = session.query(ReportPathData.report_id) \ + .join(ReportPathDataFile, + ReportPathData.report_id == + ReportPathDataFile.c.report_path_data_id) \ + .join(File, File.id != ReportPathDataFile.c.file_id) \ .filter(file_filter_q) neg_q_report = session.query(Report.id) \ .join(File, File.id != Report.file_id) \ .filter(file_filter_q) - neg_q_bugpathevent = session.query(BugPathEvent.report_id) \ - .join(File, File.id != BugPathEvent.file_id) \ - .filter(file_filter_q) + return q_report.union(q_reportpathdata) \ + .except_(neg_q_report, neg_q_reportpathdata) - neg_q_bugreportpoint = session.query(BugReportPoint.report_id) \ - .join(File, File.id != BugReportPoint.file_id) \ - .filter(file_filter_q) - neg_q_extendedreportdata = session.query(ExtendedReportData.report_id) \ - .join(File, File.id != ExtendedReportData.file_id) \ - .filter(file_filter_q) - - return q_report.union( - q_bugpathevent, - q_bugreportpoint, - q_extendedreportdata).except_( - neg_q_report, - neg_q_bugpathevent, - neg_q_bugreportpoint, - neg_q_extendedreportdata) - - -def get_reports_by_bugpath_filter(session, file_filter_q) -> Set[int]: +def get_reports_by_bugpath_filter( + session: DBSession, + file_filter_q +) -> Set[int]: """ This function returns a query for report IDs that are related to any file described by the query in the second parameter, either because their bug @@ -615,22 +600,14 @@ def get_reports_by_bugpath_filter(session, file_filter_q) -> Set[int]: .join(File, File.id == Report.file_id) \ .filter(file_filter_q) - q_bugpathevent = session.query(BugPathEvent.report_id) \ - .join(File, File.id == BugPathEvent.file_id) \ - .filter(file_filter_q) - - q_bugreportpoint = session.query(BugReportPoint.report_id) \ - .join(File, File.id == BugReportPoint.file_id) \ - .filter(file_filter_q) - - q_extendedreportdata = session.query(ExtendedReportData.report_id) \ - .join(File, File.id == ExtendedReportData.file_id) \ + q_reportpathdata = session.query(ReportPathData.report_id) \ + .join(ReportPathDataFile, + ReportPathData.report_id == + ReportPathDataFile.c.report_path_data_id) \ + .join(File, File.id == ReportPathDataFile.c.file_id) \ .filter(file_filter_q) - return q_report.union( - q_bugpathevent, - q_extendedreportdata, - q_bugreportpoint) + return q_report.union(q_reportpathdata) def get_reports_by_components(session, @@ -890,52 +867,38 @@ def process_run_filter(session, query, run_filter): return query -def get_report_details(session, report_ids): +def get_report_details( + session: DBSession, + report_ids: List[int] +) -> Dict[int, ReportDetails]: """ Returns report details for the given report ids. """ details = {} - # Get bug path events. - bug_path_events = session.query(BugPathEvent, File.filepath) \ - .filter(BugPathEvent.report_id.in_(report_ids)) \ - .outerjoin(File, - File.id == BugPathEvent.file_id) \ - .order_by(BugPathEvent.report_id, BugPathEvent.order) - bug_events_list = defaultdict(list) - for event, file_path in bug_path_events: - report_id = event.report_id - event = bugpathevent_db_to_api(event) - event.filePath = file_path - bug_events_list[report_id].append(event) - - # Get bug report points. - bug_report_points = session.query(BugReportPoint, File.filepath) \ - .filter(BugReportPoint.report_id.in_(report_ids)) \ - .outerjoin(File, - File.id == BugReportPoint.file_id) \ - .order_by(BugReportPoint.report_id, BugReportPoint.order) - bug_point_list = defaultdict(list) - for bug_point, file_path in bug_report_points: - report_id = bug_point.report_id - bug_point = bugreportpoint_db_to_api(bug_point) - bug_point.filePath = file_path - bug_point_list[report_id].append(bug_point) - - # Get extended report data. extended_data_list = defaultdict(list) - q = session.query(ExtendedReportData, File.filepath) \ - .filter(ExtendedReportData.report_id.in_(report_ids)) \ - .outerjoin(File, - File.id == ExtendedReportData.file_id) - for data, file_path in q: - report_id = data.report_id - extended_data = extended_data_db_to_api(data) - extended_data.filePath = file_path - extended_data_list[report_id].append(extended_data) + report_path_data = session.query(ReportPathData) \ + .filter(ReportPathData.report_id.in_(report_ids)) + + for rpd in report_path_data: + files = {f.id: f.filepath for f in rpd.files} + + for pd in rpd.path_data: + if pd["type"] == "event": + event = bugpathevent_db_to_api(pd) + event.filePath = files[pd["fid"]] + bug_events_list[rpd.report_id].append(event) + elif pd["type"] == "path": + bug_point = bugreportpoint_db_to_api(pd) + bug_point.filePath = files[pd["fid"]] + bug_point_list[rpd.report_id].append(bug_point) + else: + extended_data = extended_data_db_to_api(pd) + extended_data.filePath = files[pd["fid"]] + extended_data_list[rpd.report_id].append(extended_data) # Get Comments for report data comment_data_list = defaultdict(list) @@ -958,34 +921,34 @@ def get_report_details(session, report_ids): return details -def bugpathevent_db_to_api(bpe): +def bugpathevent_db_to_api(bpe: Dict) -> ttypes.BugPathEvent: return ttypes.BugPathEvent( - startLine=bpe.line_begin, - startCol=bpe.col_begin, - endLine=bpe.line_end, - endCol=bpe.col_end, - msg=bpe.msg, - fileId=bpe.file_id) + startLine=bpe["from"]["row"], + startCol=bpe["from"]["col"], + endLine=bpe["to"]["row"], + endCol=bpe["to"]["col"], + msg=bpe["msg"], + fileId=bpe["fid"]) -def bugreportpoint_db_to_api(brp): - return BugPathPos( - startLine=brp.line_begin, - startCol=brp.col_begin, - endLine=brp.line_end, - endCol=brp.col_end, - fileId=brp.file_id) +def bugreportpoint_db_to_api(brp: Dict) -> ttypes.BugPathPos: + return ttypes.BugPathPos( + startLine=brp["from"]["row"], + startCol=brp["from"]["col"], + endLine=brp["to"]["row"], + endCol=brp["to"]["col"], + fileId=brp["fid"]) -def extended_data_db_to_api(erd): +def extended_data_db_to_api(erd: Dict) -> ttypes.ExtendedReportData: return ttypes.ExtendedReportData( - type=report_extended_data_type_enum(erd.type), - startLine=erd.line_begin, - startCol=erd.col_begin, - endLine=erd.line_end, - endCol=erd.col_end, - message=erd.message, - fileId=erd.file_id) + type=report_extended_data_type_enum(erd["type"]), + startLine=erd["from"]["row"], + startCol=erd["from"]["col"], + endLine=erd["to"]["row"], + endCol=erd["to"]["col"], + message=erd["msg"], + fileId=erd["fid"]) def comment_data_db_to_api(comm): diff --git a/web/server/codechecker_server/database/db_cleanup.py b/web/server/codechecker_server/database/db_cleanup.py index e1c609aa0b..55d0c8e501 100644 --- a/web/server/codechecker_server/database/db_cleanup.py +++ b/web/server/codechecker_server/database/db_cleanup.py @@ -22,10 +22,10 @@ from .database import DBSession from .run_db_model import \ AnalysisInfo, \ - BugPathEvent, BugReportPoint, \ Comment, Checker, \ File, FileContent, \ - Report, ReportAnalysisInfo, RunHistoryAnalysisInfo, RunLock + Report, ReportAnalysisInfo, ReportPathDataFile, RunHistoryAnalysisInfo, \ + RunLock LOG = get_logger('server') RUN_LOCK_TIMEOUT_IN_DATABASE = 30 * 60 # 30 minutes. @@ -88,19 +88,16 @@ def remove_unused_files(product): LOG.debug("[%s] Garbage collection of dangling files started...", product.endpoint) try: - bpe_files = session.query(BugPathEvent.file_id) \ - .group_by(BugPathEvent.file_id) - brp_files = session.query(BugReportPoint.file_id) \ - .group_by(BugReportPoint.file_id) + files = session.query(ReportPathDataFile.c.file_id) \ + .group_by(ReportPathDataFile.c.file_id) files_to_delete = session.query(File.id) \ - .filter(File.id.notin_(bpe_files), File.id.notin_(brp_files)) + .filter(File.id.notin_(files)) files_to_delete = map(lambda x: x[0], files_to_delete) total_count = 0 for chunk in util.chunks(iter(files_to_delete), chunk_size): - q = session.query(File) \ - .filter(File.id.in_(chunk)) + q = session.query(File).filter(File.id.in_(chunk)) count = q.delete(synchronize_session=False) if count: total_count += count diff --git a/web/server/codechecker_server/database/run_db_model.py b/web/server/codechecker_server/database/run_db_model.py index f240ae7826..d96c6a7700 100644 --- a/web/server/codechecker_server/database/run_db_model.py +++ b/web/server/codechecker_server/database/run_db_model.py @@ -9,13 +9,16 @@ SQLAlchemy ORM model for the analysis run storage database. """ from datetime import datetime, timedelta +import json from math import ceil import os from typing import Optional +import zlib from sqlalchemy import Boolean, Column, DateTime, Enum, ForeignKey, Integer, \ LargeBinary, MetaData, String, UniqueConstraint, Table, Text, JSON from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import relationship from sqlalchemy.sql.expression import true, false @@ -263,106 +266,53 @@ def __init__(self, filepath, content_hash, remote_url, tracking_branch): self.tracking_branch = tracking_branch -class BugPathEvent(Base): - __tablename__ = 'bug_path_events' - - line_begin = Column(Integer) - col_begin = Column(Integer) - line_end = Column(Integer) - col_end = Column(Integer) - - order = Column(Integer, primary_key=True) - - msg = Column(String) - file_id = Column(Integer, ForeignKey('files.id', deferrable=True, - initially="DEFERRED", - ondelete='CASCADE'), index=True) - report_id = Column(Integer, ForeignKey('reports.id', deferrable=True, - initially="DEFERRED", - ondelete='CASCADE'), - index=True, - primary_key=True) - - def __init__(self, line_begin, col_begin, line_end, col_end, - order, msg, file_id, report_id): - self.line_begin, self.col_begin, self.line_end, self.col_end = \ - line_begin, col_begin, line_end, col_end - - self.order = order - self.msg = msg - self.file_id = file_id - self.report_id = report_id - - -class BugReportPoint(Base): - __tablename__ = 'bug_report_points' - - line_begin = Column(Integer) - col_begin = Column(Integer) - line_end = Column(Integer) - col_end = Column(Integer) - - order = Column(Integer, primary_key=True) - - file_id = Column(Integer, ForeignKey('files.id', deferrable=True, - initially="DEFERRED", - ondelete='CASCADE'), index=True) - report_id = Column(Integer, ForeignKey('reports.id', deferrable=True, - initially="DEFERRED", - ondelete='CASCADE'), - index=True, - primary_key=True) - - def __init__(self, line_begin, col_begin, line_end, col_end, - order, file_id, report_id): - self.line_begin, self.col_begin, self.line_end, self.col_end = \ - line_begin, col_begin, line_end, col_end - - self.order = order - self.file_id = file_id - self.report_id = report_id +ReportPathDataFile = Table( + 'report_path_data_files', + Base.metadata, + Column( + 'report_path_data_id', + Integer, + ForeignKey('report_path_data.report_id', + deferrable=True, + initially="DEFERRED", + ondelete="CASCADE"), + index=True), + Column('file_id', Integer, ForeignKey('files.id')) +) -class ExtendedReportData(Base): +class ReportPathData(Base): """ - Store extra information which can help to understand or fix a report. + Store bug path positions, bug events, macro expansions, fix-its, etc. """ - __tablename__ = 'extended_report_data' - - id = Column(Integer, autoincrement=True, primary_key=True) - - report_id = Column(Integer, ForeignKey('reports.id', deferrable=True, - initially="DEFERRED", - ondelete='CASCADE'), - index=True) + __tablename__ = 'report_path_data' - file_id = Column(Integer, ForeignKey('files.id', deferrable=True, - initially="DEFERRED", - ondelete='CASCADE'), index=True) + report_id = Column( + Integer, + ForeignKey( + 'reports.id', + deferrable=True, + initially="DEFERRED", + ondelete='CASCADE'), + primary_key=True) - type = Column(Enum('note', - 'macro', - 'fixit', - name='extended_data_type')) + _path_data = Column("path_data", LargeBinary) - line_begin = Column(Integer) - col_begin = Column(Integer) - line_end = Column(Integer) - col_end = Column(Integer) + @hybrid_property + def path_data(self): + return json.loads(zlib.decompress(self._path_data).decode("utf-8")) - message = Column(String) + @path_data.setter + def path_data(self, data): + self._path_data = zlib.compress( + json.dumps(data).encode("utf-8"), + zlib.Z_BEST_COMPRESSION) - def __init__(self, line_begin, col_begin, line_end, col_end, - message, file_id, report_id, data_type): + files = relationship("File", secondary=ReportPathDataFile) - self.line_begin = line_begin - self.col_begin = col_begin - self.line_end = line_end - self.col_end = col_end - self.message = message - self.file_id = file_id + def __init__(self, report_id, path_data): self.report_id = report_id - self.type = data_type + self.path_data = path_data ReportAnalysisInfo = Table( diff --git a/web/server/codechecker_server/migrations/report/versions/ffb641e13b92_report_path_data_table.py b/web/server/codechecker_server/migrations/report/versions/ffb641e13b92_report_path_data_table.py new file mode 100644 index 0000000000..92d7483e8c --- /dev/null +++ b/web/server/codechecker_server/migrations/report/versions/ffb641e13b92_report_path_data_table.py @@ -0,0 +1,329 @@ +""" +Report path data table + +Revision ID: ffb641e13b92 +Revises: 24c9660f82b1 +Create Date: 2026-06-16 02:50:24.445500 +""" + +from logging import getLogger + +from alembic import op +import sqlalchemy as sa + +import json +import zlib +from collections import defaultdict + +from codechecker_common import util + + +# Revision identifiers, used by Alembic. +revision = 'ffb641e13b92' +down_revision = '24c9660f82b1' +branch_labels = None +depends_on = None + + +def upgrade(): + LOG = getLogger("migration/report") + # ### commands auto generated by Alembic - please adjust! ### + report_path_data_table = op.create_table( + 'report_path_data', + sa.Column('report_id', sa.Integer(), nullable=False), + sa.Column('path_data', sa.LargeBinary(), nullable=True), + sa.ForeignKeyConstraint( + ['report_id'], + ['reports.id'], + name=op.f('fk_report_path_data_report_id_reports'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.PrimaryKeyConstraint('report_id', name=op.f('pk_report_path_data')) + ) + report_path_data_files_table = op.create_table( + 'report_path_data_files', + sa.Column('report_path_data_id', sa.Integer(), nullable=True), + sa.Column('file_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ['file_id'], + ['files.id'], + name=op.f('fk_report_path_data_files_file_id_files')), + sa.ForeignKeyConstraint( + ['report_path_data_id'], + ['report_path_data.report_id'], + name=op.f( + 'fk_report_path_data_files_report_path_data_id_' + 'report_path_data'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True) + ) + + conn = op.get_bind() + + report_ids = conn.execute(sa.text("""SELECT id FROM reports""")) + + for report_id_chunk in util.chunks(report_ids, 100_000): + report_id_chunk = list(map(lambda x: x[0], report_id_chunk)) + + conn.execute(sa.text("BEGIN")) + + report_path_data = defaultdict(lambda: { + 'path_data': [], + 'used_file_ids': set() + }) + + bug_report_points = conn.execute(sa.text(f""" + SELECT * FROM bug_report_points + WHERE report_id IN ({','.join(map(str, report_id_chunk))}) + ORDER BY report_id, "order" + """)) + + for brp in bug_report_points: + report_path_data[brp.report_id]['path_data'].append({ + "from": { + "row": brp.line_begin, + "col": brp.col_begin + }, + "to": { + "row": brp.line_end, + "col": brp.col_end + }, + "type": "path", + "fid": brp.file_id + }) + report_path_data[brp.report_id]['used_file_ids'].add(brp.file_id) + + bug_path_events = conn.execute(sa.text(f""" + SELECT * FROM bug_path_events + WHERE report_id IN ({','.join(map(str, report_id_chunk))}) + ORDER BY report_id, "order" + """)) + + for bpe in bug_path_events: + report_path_data[bpe.report_id]['path_data'].append({ + "from": { + "row": bpe.line_begin, + "col": bpe.col_begin + }, + "to": { + "row": bpe.line_end, + "col": bpe.col_end + }, + "type": "event", + "msg": bpe.msg, + "fid": bpe.file_id + }) + report_path_data[bpe.report_id]['used_file_ids'].add(bpe.file_id) + + extended_report_data = conn.execute(sa.text(f""" + SELECT * FROM extended_report_data + WHERE report_id in ({','.join(map(str, report_id_chunk))}) + """)) + + for erd in extended_report_data: + report_path_data[erd.report_id]['path_data'].append({ + "from": { + "row": erd.line_begin, + "col": erd.col_begin + }, + "to": { + "row": erd.line_end, + "col": erd.col_end + }, + "type": erd.type, + "msg": erd.message, + "fid": erd.file_id + }) + report_path_data[erd.report_id]['used_file_ids'].add(erd.file_id) + + report_data_content = [] + report_data_file_content = [] + + for report_id, data in report_path_data.items(): + compressed_data = zlib.compress( + json.dumps(data['path_data']).encode('utf-8'), + zlib.Z_BEST_COMPRESSION) + + report_data_content.append({ + 'report_id': report_id, + 'path_data': compressed_data + }) + + report_data_file_content.extend([{ + 'report_path_data_id': report_id, + 'file_id': fid + } for fid in data['used_file_ids']]) + + conn.execute( + sa.insert(report_path_data_table), + report_data_content) + + conn.execute( + sa.insert(report_path_data_files_table), + report_data_file_content) + + conn.execute(sa.text("COMMIT")) + + op.create_index( + op.f('ix_report_path_data_files_report_path_data_id'), + 'report_path_data_files', + ['report_path_data_id'], + unique=False) + op.drop_index( + op.f('ix_extended_report_data_file_id'), + table_name='extended_report_data') + op.drop_index( + op.f('ix_extended_report_data_report_id'), + table_name='extended_report_data') + op.drop_table('extended_report_data') + op.drop_index( + op.f('ix_bug_report_points_file_id'), + table_name='bug_report_points') + op.drop_index( + op.f('ix_bug_report_points_report_id'), + table_name='bug_report_points') + op.drop_table('bug_report_points') + op.drop_index( + op.f('ix_bug_path_events_file_id'), + table_name='bug_path_events') + op.drop_index( + op.f('ix_bug_path_events_report_id'), + table_name='bug_path_events') + op.drop_table('bug_path_events') + + # ### end Alembic commands ### + + +def downgrade(): + # TODO: Currently we're not supporting the downgrade of DB schema version. + # This is a script generated by Alembic, but the old tables are not + # populated by data from the new schema. + LOG = getLogger("migration/report") + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + 'bug_path_events', + sa.Column('line_begin', sa.INTEGER(), nullable=True), + sa.Column('col_begin', sa.INTEGER(), nullable=True), + sa.Column('line_end', sa.INTEGER(), nullable=True), + sa.Column('col_end', sa.INTEGER(), nullable=True), + sa.Column('order', sa.INTEGER(), nullable=False), + sa.Column('msg', sa.VARCHAR(), nullable=True), + sa.Column('file_id', sa.INTEGER(), nullable=True), + sa.Column('report_id', sa.INTEGER(), nullable=False), + sa.ForeignKeyConstraint( + ['file_id'], + ['files.id'], + name=op.f('fk_bug_path_events_file_id_files'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.ForeignKeyConstraint( + ['report_id'], + ['reports.id'], + name=op.f('fk_bug_path_events_report_id_reports'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.PrimaryKeyConstraint( + 'order', + 'report_id', + name=op.f('pk_bug_path_events')) + ) + op.create_index( + op.f('ix_bug_path_events_report_id'), + 'bug_path_events', + ['report_id'], + unique=False) + op.create_index( + op.f('ix_bug_path_events_file_id'), + 'bug_path_events', + ['file_id'], + unique=False) + op.create_table( + 'bug_report_points', + sa.Column('line_begin', sa.INTEGER(), nullable=True), + sa.Column('col_begin', sa.INTEGER(), nullable=True), + sa.Column('line_end', sa.INTEGER(), nullable=True), + sa.Column('col_end', sa.INTEGER(), nullable=True), + sa.Column('order', sa.INTEGER(), nullable=False), + sa.Column('file_id', sa.INTEGER(), nullable=True), + sa.Column('report_id', sa.INTEGER(), nullable=False), + sa.ForeignKeyConstraint( + ['file_id'], + ['files.id'], + name=op.f('fk_bug_report_points_file_id_files'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.ForeignKeyConstraint( + ['report_id'], + ['reports.id'], + name=op.f('fk_bug_report_points_report_id_reports'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.PrimaryKeyConstraint( + 'order', + 'report_id', + name=op.f('pk_bug_report_points')) + ) + op.create_index( + op.f('ix_bug_report_points_report_id'), + 'bug_report_points', + ['report_id'], + unique=False) + op.create_index( + op.f('ix_bug_report_points_file_id'), + 'bug_report_points', + ['file_id'], + unique=False) + op.create_table( + 'extended_report_data', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('report_id', sa.INTEGER(), nullable=False), + sa.Column('file_id', sa.INTEGER(), nullable=True), + sa.Column('type', sa.VARCHAR(length=5), nullable=True), + sa.Column('line_begin', sa.INTEGER(), nullable=True), + sa.Column('col_begin', sa.INTEGER(), nullable=True), + sa.Column('line_end', sa.INTEGER(), nullable=True), + sa.Column('col_end', sa.INTEGER(), nullable=True), + sa.Column('message', sa.VARCHAR(), nullable=True), + sa.ForeignKeyConstraint( + ['file_id'], + ['files.id'], + name=op.f('fk_extended_report_data_file_id_files'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.ForeignKeyConstraint( + ['report_id'], + ['reports.id'], + name=op.f('fk_extended_report_data_report_id_reports'), + ondelete='CASCADE', + initially='DEFERRED', + deferrable=True), + sa.PrimaryKeyConstraint( + 'id', + name=op.f('pk_extended_report_data')) + ) + op.create_index( + op.f('ix_extended_report_data_report_id'), + 'extended_report_data', + ['report_id'], + unique=False) + op.create_index( + op.f('ix_extended_report_data_file_id'), + 'extended_report_data', + ['file_id'], + unique=False) + op.drop_index( + op.f('ix_report_path_data_files_report_path_data_id'), + table_name='report_path_data_files') + op.drop_table('report_path_data_files') + op.drop_index( + op.f('ix_report_path_data_report_id'), + table_name='report_path_data') + op.drop_table('report_path_data') + # ### end Alembic commands ###