Merge pull request #25964 from frappe/version-14-hotfix

chore: release v14

frappe/core/doctype/prepared_report/prepared_report.py

@@ -7,6 +7,7 @@
 from rq import get_current_job
 
 import frappe
+from frappe.database.utils import dangerously_reconnect_on_connection_abort
 from frappe.desk.form.load import get_attachments
 from frappe.desk.query_report import generate_report_result
 from frappe.model.document import Document
@@ -82,7 +83,8 @@ def generate_report(prepared_report):
 		instance.status = "Completed"
 	except Exception:
 		instance.status = "Error"
-		instance.error_message = frappe.get_traceback()
+		instance.error_message = frappe.get_traceback(with_context=True)
+		_save_instance(instance)  # we need to ensure that error gets stored
 
 	instance.report_end_time = frappe.utils.now()
 	instance.save(ignore_permissions=True)
@@ -99,6 +101,11 @@ def update_job_id(prepared_report, job_id):
 	frappe.db.commit()
 
 
+@dangerously_reconnect_on_connection_abort
+def _save_instance(instance):
+	instance.save(ignore_permissions=True)
+
+
 @frappe.whitelist()
 def make_prepared_report(report_name, filters=None):
 	"""run reports in background"""

frappe/custom/doctype/customize_form/customize_form.js

@@ -134,6 +134,14 @@ frappe.ui.form.on("Customize Form", {
 			const is_autoname_autoincrement = frm.doc.autoname === "autoincrement";
 			frm.set_df_property("naming_rule", "hidden", is_autoname_autoincrement);
 			frm.set_df_property("autoname", "read_only", is_autoname_autoincrement);
+
+			frm.add_custom_button(
+				__("Trim Table"),
+				function () {
+					frm.trigger("trim_table");
+				},
+				__("Actions")
+			);
 		}
 
 		frm.events.setup_export(frm);
@@ -156,6 +164,40 @@ frappe.ui.form.on("Customize Form", {
 		}
 	},
 
+	async trim_table(frm) {
+		let dropped_columns = await frappe.xcall(
+			"frappe.custom.doctype.customize_form.customize_form.get_orphaned_columns",
+			{ doctype: frm.doc.doc_type }
+		);
+
+		if (!dropped_columns?.length) {
+			frappe.toast(__("This doctype has no orphan fields to trim"));
+			return;
+		}
+		let msg = __(
+			"Warning: DATA LOSS IMMINENT! Proceeding will permanently delete following database columns from doctype {0}:",
+			[frm.doc.doc_type.bold()]
+		);
+		msg += "<ol>" + dropped_columns.map((col) => `<li>${col}</li>`).join("") + "</ol>";
+		msg += __("This action is irreversible. Do you wish to continue?");
+
+		frappe.confirm(msg, () => {
+			return frm.call({
+				doc: frm.doc,
+				method: "trim_table",
+				callback: function (r) {
+					if (!r.exc) {
+						frappe.show_alert({
+							message: __("Table Trimmed"),
+							indicator: "green",
+						});
+						frappe.customize_form.clear_locals_and_refresh(frm);
+					}
+				},
+			});
+		});
+	},
+
 	setup_export(frm) {
 		if (frappe.boot.developer_mode) {
 			frm.add_custom_button(

frappe/custom/doctype/customize_form/customize_form.py

@@ -21,6 +21,7 @@
 from frappe.model import core_doctypes_list, no_value_fields
 from frappe.model.docfield import supports_translation
 from frappe.model.document import Document
+from frappe.model.meta import trim_table
 from frappe.utils import cint
 
 
@@ -553,6 +554,19 @@ def reset_to_defaults(self):
 		reset_customization(self.doc_type)
 		self.fetch_to_customize()
 
+	@frappe.whitelist()
+	def trim_table(self):
+		"""Removes database fields that don't exist in the doctype.
+
+		This may be needed as maintenance since removing a field in a DocType
+		doesn't automatically delete the db field.
+		"""
+		if not self.doc_type:
+			return
+
+		trim_table(self.doc_type, dry_run=False)
+		self.fetch_to_customize()
+
 	@classmethod
 	def allow_fieldtype_change(self, old_type: str, new_type: str) -> bool:
 		"""allow type change, if both old_type and new_type are in same field group.
@@ -565,6 +579,13 @@ def in_field_group(group):
 		return any(map(in_field_group, ALLOWED_FIELDTYPE_CHANGE))
 
 
+@frappe.whitelist()
+def get_orphaned_columns(doctype: str):
+	frappe.only_for("System Manager")
+	frappe.db.begin(read_only=True)  # Avoid any potential bug from writing to db
+	return trim_table(doctype, dry_run=True)
+
+
 def reset_customization(doctype):
 	setters = frappe.get_all(
 		"Property Setter",

frappe/database/database.py

@@ -1311,7 +1311,7 @@ def escape(s, percent=True):
 
 	@staticmethod
 	def is_column_missing(e):
-		return frappe.db.is_missing_column(e)
+		raise NotImplementedError
 
 	def get_descendants(self, doctype, name):
 		"""Return descendants of the group node in tree"""

frappe/database/mariadb/database.py

@@ -93,6 +93,10 @@ def is_unique_key_violation(e: pymysql.Error) -> bool:
 			and isinstance(e, pymysql.IntegrityError)
 		)
 
+	@staticmethod
+	def is_interface_error(e: pymysql.Error):
+		return isinstance(e, pymysql.InterfaceError)
+
 
 class MariaDBConnectionUtil:
 	def get_connection(self):

frappe/database/postgres/database.py

@@ -13,6 +13,7 @@
 	UNIQUE_VIOLATION,
 )
 from psycopg2.errors import (
+	InterfaceError,
 	LockNotAvailable,
 	ReadOnlySqlTransaction,
 	SequenceGeneratorLimitExceeded,
@@ -112,6 +113,10 @@ def is_statement_timeout(e):
 	def is_data_too_long(e):
 		return getattr(e, "pgcode", None) == STRING_DATA_RIGHT_TRUNCATION
 
+	@staticmethod
+	def is_interface_error(e):
+		return isinstance(e, InterfaceError)
+
 
 class PostgresDatabase(PostgresExceptionUtil, Database):
 	REGEX_CHARACTER = "~"

frappe/database/utils.py

@@ -1,15 +1,14 @@
 # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
-from functools import cached_property
-from types import NoneType
+from functools import cached_property, wraps
 
 import frappe
 from frappe.query_builder.builder import MariaDB, Postgres
 from frappe.query_builder.functions import Function
 
 Query = str | MariaDB | Postgres
-QueryValues = tuple | list | dict | NoneType
+QueryValues = tuple | list | dict | None
 
 EmptyQueryValues = object()
 FallBackDateTimeStr = "0001-01-01 00:00:00.000000"
@@ -72,3 +71,26 @@ def __init__(self, query, values) -> None:
 
 	def _setup(self) -> str:
 		return frappe.db.mogrify(self.query, self.values)
+
+
+def dangerously_reconnect_on_connection_abort(func):
+	"""Reconnect on connection failure.
+
+	As the name suggest, it's dangerous to use this function as it will NOT restore DB transaction
+	so make sure you're using it right.
+
+	Ideal use case: Some kinda logging or final steps in a background jobs. Anything more than that
+	will risk bugs from DB transactions.
+	"""
+
+	@wraps(func)
+	def wrapper(*args, **kwargs):
+		try:
+			return func(*args, **kwargs)
+		except Exception as e:
+			if frappe.db.is_interface_error(e):
+				frappe.db.connect()
+				return func(*args, **kwargs)
+			raise
+
+	return wrapper