feat: clean up message_files table first before proceeding to find orphaned files (#19035)

api/commands.py

     click.echo(click.style("Clear free plan tenant expired logs completed.", fg="green"))
 
 
+@click.option("-f", "--force", is_flag=True, help="Skip user confirmation and force the command to execute.")
 @click.command("clear-orphaned-file-records", help="Clear orphaned file records.")
-def clear_orphaned_file_records():
+def clear_orphaned_file_records(force: bool):
     """
     Clear orphaned file records in the database.
     """
 
     # notify user and ask for confirmation
     click.echo(
-        click.style("This command will find and delete orphaned file records in the following tables:", fg="yellow")
+        click.style(
+            "This command will first find and delete orphaned file records from the message_files table,", fg="yellow"
+        )
+    )
+    click.echo(
+        click.style(
+            "and then it will find and delete orphaned file records in the following tables:",
+            fg="yellow",
+        )
     )
     for files_table in files_tables:
         click.echo(click.style(f"- {files_table['table']}", fg="yellow"))
             fg="yellow",
         )
     )
-    click.confirm("Do you want to proceed?", abort=True)
+    if not force:
+        click.confirm("Do you want to proceed?", abort=True)
 
     # start the cleanup process
     click.echo(click.style("Starting orphaned file records cleanup.", fg="white"))
 
+    # clean up the orphaned records in the message_files table where message_id doesn't exist in messages table
+    try:
+        click.echo(
+            click.style("- Listing message_files records where message_id doesn't exist in messages table", fg="white")
+        )
+        query = (
+            "SELECT mf.id, mf.message_id "
+            "FROM message_files mf LEFT JOIN messages m ON mf.message_id = m.id "
+            "WHERE m.id IS NULL"
+        )
+        orphaned_message_files = []
+        with db.engine.begin() as conn:
+            rs = conn.execute(db.text(query))
+            for i in rs:
+                orphaned_message_files.append({"id": str(i[0]), "message_id": str(i[1])})
+
+        if orphaned_message_files:
+            click.echo(click.style(f"Found {len(orphaned_message_files)} orphaned message_files records:", fg="white"))
+            for record in orphaned_message_files:
+                click.echo(click.style(f"  - id: {record['id']}, message_id: {record['message_id']}", fg="black"))
+
+            if not force:
+                click.confirm(
+                    (
+                        f"Do you want to proceed "
+                        f"to delete all {len(orphaned_message_files)} orphaned message_files records?"
+                    ),
+                    abort=True,
+                )
+
+            click.echo(click.style("- Deleting orphaned message_files records", fg="white"))
+            query = "DELETE FROM message_files WHERE id IN :ids"
+            with db.engine.begin() as conn:
+                conn.execute(db.text(query), {"ids": tuple([record["id"] for record in orphaned_message_files])})
+            click.echo(
+                click.style(f"Removed {len(orphaned_message_files)} orphaned message_files records.", fg="green")
+            )
+        else:
+            click.echo(click.style("No orphaned message_files records found. There is nothing to delete.", fg="green"))
+    except Exception as e:
+        click.echo(click.style(f"Error deleting orphaned message_files records: {str(e)}", fg="red"))
+
+    # clean up the orphaned records in the rest of the *_files tables
     try:
         # fetch file id and keys from each table
         all_files_in_tables = []
     click.echo(click.style(f"Found {len(orphaned_files)} orphaned file records.", fg="white"))
     for file in orphaned_files:
         click.echo(click.style(f"- orphaned file id: {file}", fg="black"))
-    click.confirm(f"Do you want to proceed to delete all {len(orphaned_files)} orphaned file records?", abort=True)
+    if not force:
+        click.confirm(f"Do you want to proceed to delete all {len(orphaned_files)} orphaned file records?", abort=True)
 
     # delete orphaned records for each file
     try:
     click.echo(click.style(f"Removed {len(orphaned_files)} orphaned file records.", fg="green"))
 
 
+@click.option("-f", "--force", is_flag=True, help="Skip user confirmation and force the command to execute.")
 @click.command("remove-orphaned-files-on-storage", help="Remove orphaned files on the storage.")
-def remove_orphaned_files_on_storage():
+def remove_orphaned_files_on_storage(force: bool):
     """
     Remove orphaned files on the storage.
     """
             fg="yellow",
         )
     )
-    click.confirm("Do you want to proceed?", abort=True)
+    if not force:
+        click.confirm("Do you want to proceed?", abort=True)
 
     # start the cleanup process
     click.echo(click.style("Starting orphaned files cleanup.", fg="white"))
     click.echo(click.style(f"Found {len(orphaned_files)} orphaned files.", fg="white"))
     for file in orphaned_files:
         click.echo(click.style(f"- orphaned file: {file}", fg="black"))
-    click.confirm(f"Do you want to proceed to remove all {len(orphaned_files)} orphaned files?", abort=True)
+    if not force:
+        click.confirm(f"Do you want to proceed to remove all {len(orphaned_files)} orphaned files?", abort=True)
 
     # delete orphaned files
     removed_files = 0