Merge pull request #25773 from frappe/version-15-hotfix

chore: release v15

.github/helper/install.sh

@@ -2,9 +2,11 @@
 set -e
 cd ~ || exit
 
-echo "Setting Up Bench..."
-
+echo "::group::Install Bench"
 pip install frappe-bench
+echo "::endgroup::"
+
+echo "::group::Init Bench"
 bench -v init frappe-bench --skip-assets --python "$(which python)" --frappe-path "${GITHUB_WORKSPACE}"
 cd ./frappe-bench || exit
 
@@ -13,9 +15,9 @@ if [ "$TYPE" == "ui" ]
 then
   bench -v setup requirements --node;
 fi
+echo "::endgroup::"
 
-echo "Setting Up Sites & Database..."
-
+echo "::group::Create Test Site"
 mkdir ~/frappe-bench/sites/test_site
 cp "${GITHUB_WORKSPACE}/.github/helper/db/$DB.json" ~/frappe-bench/sites/test_site/site_config.json
 
@@ -35,9 +37,9 @@ then
   echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe" -U postgres;
   echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe WITH PASSWORD 'test_frappe'" -U postgres;
 fi
+echo "::endgroup::"
 
-echo "Setting Up Procfile..."
-
+echo "::group::Modify processes"
 sed -i 's/^watch:/# watch:/g' Procfile
 sed -i 's/^schedule:/# schedule:/g' Procfile
 
@@ -51,11 +53,11 @@ if [ "$TYPE" == "ui" ]
 then
   sed -i 's/^web: bench serve/web: bench serve --with-coverage/g' Procfile
 fi
-
-echo "Starting Bench..."
+echo "::endgroup::"
 
 bench start &> ~/frappe-bench/bench_start.log &
 
+echo "::group::Install site"
 if [ "$TYPE" == "server" ]
 then
   CI=Yes bench build --app frappe &
@@ -69,3 +71,4 @@ then
   # wait till assets are built succesfully
   wait $build_pid
 fi
+echo "::endgroup::"

.github/helper/install_dependencies.sh

@@ -3,6 +3,7 @@ set -e
 
 echo "Setting Up System Dependencies..."
 
+echo "::group::apt packages"
 sudo apt update
 sudo apt remove mysql-server mysql-client
 sudo apt install libcups2-dev redis-server mariadb-client-10.6
@@ -12,3 +13,4 @@ install_wkhtmltopdf() {
   sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb
 }
 install_wkhtmltopdf &
+echo "::endgroup::"

frappe/commands/site.py

@@ -243,8 +243,28 @@ def restore_backup(
 	admin_password,
 	force,
 ):
+	from pathlib import Path
+
 	from frappe.installer import _new_site, is_downgrade, is_partial, validate_database_sql
 
+	# Check for the backup file in the backup directory, as well as the main bench directory
+	dirs = (f"{site}/private/backups", "..")
+
+	# Try to resolve path to the file if we can't find it directly
+	if not Path(sql_file_path).exists():
+		click.secho(
+			f"File {sql_file_path} not found. Trying to check in alternative directories.", fg="yellow"
+		)
+		for dir in dirs:
+			potential_path = Path(dir) / Path(sql_file_path)
+			if potential_path.exists():
+				sql_file_path = str(potential_path.resolve())
+				click.secho(f"File {sql_file_path} found.", fg="green")
+				break
+		else:
+			click.secho(f"File {sql_file_path} not found.", fg="red")
+			sys.exit(1)
+
 	if is_partial(sql_file_path):
 		click.secho(
 			"Partial Backup file detected. You cannot use a partial file to restore a Frappe site.",

frappe/core/doctype/communication/communication.json

@@ -401,7 +401,8 @@
  "icon": "fa fa-comment",
  "idx": 1,
  "links": [],
- "modified": "2023-11-27 20:38:27.467076",
+ "make_attachments_public": 1,
+ "modified": "2024-02-09 12:10:01.200845",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Communication",

frappe/core/doctype/data_import/data_import_list.js

@@ -27,10 +27,6 @@ frappe.listview_settings["Data Import"] = {
 		if (imports_in_progress.includes(doc.name)) {
 			status = "In Progress";
 		}
-		if (status === "Pending") {
-			status = "Not Started";
-		}
-
 		return [__(status), colors[status], "status,=," + doc.status];
 	},
 	formatters: {

frappe/core/doctype/data_import/importer.py

@@ -102,9 +102,13 @@ def import_data(self):
 		log_index = 0
 
 		# Do not remove rows in case of retry after an error or pending data import
-		if self.data_import.status == "Partial Success" and len(import_log) >= self.data_import.payload_count:
+		if (
+			self.data_import.status in ("Partial Success", "Error")
+			and len(import_log) >= self.data_import.payload_count
+		):
 			# remove previous failures from import log only in case of retry after partial success
 			import_log = [log for log in import_log if log.get("success")]
+			frappe.db.delete("Data Import Log", {"success": 0, "data_import": self.data_import.name})
 
 		# get successfully imported rows
 		imported_rows = []
@@ -213,13 +217,21 @@ def import_data(self):
 		)
 
 		# set status
-		failures = [log for log in import_log if not log.get("success")]
-		if len(failures) == total_payload_count:
-			status = "Pending"
-		elif len(failures) > 0:
+		successes = []
+		failures = []
+		for log in import_log:
+			if log.get("success"):
+				successes.append(log)
+			else:
+				failures.append(log)
+		if len(failures) >= total_payload_count and len(successes) == 0:
+			status = "Error"
+		elif len(failures) > 0 and len(successes) > 0:
 			status = "Partial Success"
-		else:
+		elif len(successes) == total_payload_count:
 			status = "Success"
+		else:
+			status = "Pending"
 
 		if self.console:
 			self.print_import_log(import_log)

frappe/core/doctype/file/test_file.py

@@ -18,7 +18,8 @@
 	unzip_file,
 )
 from frappe.core.doctype.file.exceptions import FileTypeNotAllowed
-from frappe.core.doctype.file.utils import delete_file, get_extension
+from frappe.core.doctype.file.utils import get_extension
+from frappe.desk.form.utils import add_comment
 from frappe.exceptions import ValidationError
 from frappe.tests.utils import FrappeTestCase, change_settings
 from frappe.utils import get_files_path, set_request
@@ -722,27 +723,65 @@ def tearDown(self) -> None:
 
 class TestFileUtils(FrappeTestCase):
 	def test_extract_images_from_doc(self):
+		is_private = not frappe.get_meta("ToDo").make_attachments_public
+
 		# with filename in data URI
 		todo = frappe.get_doc(
-			{
-				"doctype": "ToDo",
-				"description": 'Test <img src="data:image/png;filename=pix.png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=">',
-			}
+			doctype="ToDo",
+			description='Test <img src="data:image/png;filename=pix.png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=">',
 		).insert()
-		self.assertTrue(frappe.db.exists("File", {"attached_to_name": todo.name}))
-		self.assertIn('<img src="/files/pix.png">', todo.description)
-		self.assertListEqual(get_attached_images("ToDo", [todo.name])[todo.name], ["/files/pix.png"])
+		self.assertTrue(frappe.db.exists("File", {"attached_to_name": todo.name, "is_private": is_private}))
+		self.assertRegex(todo.description, r"<img src=\"(.*)/files/pix\.png(.*)\">")
+		self.assertListEqual(get_attached_images("ToDo", [todo.name])[todo.name], ["/private/files/pix.png"])
 
 		# without filename in data URI
 		todo = frappe.get_doc(
-			{
-				"doctype": "ToDo",
-				"description": 'Test <img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=">',
-			}
+			doctype="ToDo",
+			description='Test <img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=">',
 		).insert()
 		filename = frappe.db.exists("File", {"attached_to_name": todo.name})
 		self.assertIn(f'<img src="{frappe.get_doc("File", filename).file_url}', todo.description)
 
+	def test_extract_images_from_comment(self):
+		"""
+		Ensure that images are extracted from comments and become private attachments.
+		"""
+		is_private = not frappe.get_meta("ToDo").make_attachments_public
+		test_doc = frappe.get_doc(doctype="ToDo", description="comment test").insert()
+		comment = add_comment(
+			"ToDo",
+			test_doc.name,
+			'<div class="ql-editor read-mode"><img src="data:image/png;filename=pix.png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="></div>',
+			frappe.session.user,
+			frappe.session.user,
+		)
+
+		self.assertTrue(
+			frappe.db.exists("File", {"attached_to_name": test_doc.name, "is_private": is_private})
+		)
+		self.assertRegex(comment.content, r"<img src=\"(.*)/files/pix\.png(.*)\">")
+
+	def test_extract_images_from_communication(self):
+		"""
+		Ensure that images are extracted from communication and become public attachments.
+		"""
+		is_private = not frappe.get_meta("Communication").make_attachments_public
+		communication = frappe.get_doc(
+			doctype="Communication",
+			communication_type="Communication",
+			communication_medium="Email",
+			content='<div class="ql-editor read-mode"><img src="data:image/png;filename=pix.png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="></div>',
+			recipients="to <to@test.com>",
+			cc=None,
+			bcc=None,
+			sender="sender@test.com",
+		).insert(ignore_permissions=True)
+
+		self.assertTrue(
+			frappe.db.exists("File", {"attached_to_name": communication.name, "is_private": is_private})
+		)
+		self.assertRegex(communication.content, r"<img src=\"(.*)/files/pix\.png(.*)\">")
+
 	def test_create_new_folder(self):
 		folder = create_new_folder("test_folder", "Home")
 		self.assertTrue(folder.is_folder)

frappe/core/doctype/file/utils.py

@@ -218,7 +218,7 @@ def get_file_name(fname: str, optional_suffix: str | None = None) -> str:
 
 def extract_images_from_doc(doc: "Document", fieldname: str):
 	content = doc.get(fieldname)
-	content = extract_images_from_html(doc, content)
+	content = extract_images_from_html(doc, content, is_private=(not doc.meta.make_attachments_public))
 	if frappe.flags.has_dataurl:
 		doc.set(fieldname, content)
 

frappe/core/doctype/server_script/server_script_list.js

@@ -15,9 +15,9 @@ function add_github_star_cta(listview) {
 			listview.github_star_banner.remove();
 		}
 
-		const message = "Loving Frappe Framework?";
+		const message = __("Loving Frappe Framework?");
 		const link = "https://github.com/frappe/frappe";
-		const cta = "Star us on GitHub";
+		const cta = __("Star us on GitHub");
 
 		listview.github_star_banner = $(`
 				<div style="position: relative;">

frappe/custom/doctype/customize_form/customize_form.js

@@ -246,7 +246,7 @@ frappe.ui.form.on("Customize Form", {
 			var fields = $.map(frm.doc.fields, function (df) {
 				return frappe.model.is_value_type(df.fieldtype) ? df.fieldname : null;
 			});
-			fields = ["", "name", "modified"].concat(fields);
+			fields = ["", "name", "creation", "modified"].concat(fields);
 			frm.set_df_property("sort_field", "options", fields);
 		}
 	},

frappe/desk/doctype/event/event.py

@@ -396,6 +396,9 @@ def add_event(e, date):
 				try:
 					getdate(date)
 				except Exception:
+					# Don't show any message to the user
+					frappe.clear_last_message()
+
 					date = date.split("-")
 					date = date[0] + "-" + str(cint(date[1]) - 1) + "-" + date[2]
 

frappe/desk/reportview.py

@@ -4,6 +4,9 @@
 """build query for doclistview and return results"""
 
 import json
+from functools import lru_cache
+
+from sql_metadata import Parser
 
 import frappe
 import frappe.permissions
@@ -102,7 +105,10 @@ def validate_fields(data):
 	wildcard = update_wildcard_field_param(data)
 
 	for field in list(data.fields or []):
-		fieldname = extract_fieldname(field)
+		fieldname = extract_fieldnames(field)[0]
+		if not fieldname:
+			raise_invalid_field(fieldname)
+
 		if is_standard(fieldname):
 			continue
 
@@ -182,23 +188,21 @@ def is_standard(fieldname):
 	return fieldname in default_fields or fieldname in optional_fields or fieldname in child_table_fields
 
 
-def extract_fieldname(field):
-	for text in (",", "/*", "#"):
-		if text in field:
-			raise_invalid_field(field)
+@lru_cache
+def extract_fieldnames(field):
+	from frappe.database.schema import SPECIAL_CHAR_PATTERN
+
+	if not SPECIAL_CHAR_PATTERN.findall(field):
+		return [field]
 
-	fieldname = field
-	for sep in (" as ", " AS "):
-		if sep in fieldname:
-			fieldname = fieldname.split(sep, 1)[0]
+	columns = Parser(f"select {field} from _dummy").columns
 
-	# certain functions allowed, extract the fieldname from the function
-	if fieldname.startswith("count(") or fieldname.startswith("sum(") or fieldname.startswith("avg("):
-		if not fieldname.strip().endswith(")"):
-			raise_invalid_field(field)
-		fieldname = fieldname.split("(", 1)[1][:-1]
+	if not columns:
+		f = field.lower()
+		if ("count(" in f or "sum(" in f or "avg(" in f) and "*" in f:
+			return ["*"]
 
-	return fieldname
+	return columns
 
 
 def get_meta_and_docfield(fieldname, data):
@@ -245,13 +249,13 @@ def get_parenttype_and_fieldname(field, data):
 		parts = field.split(".")
 		parenttype = parts[0]
 		fieldname = parts[1]
-		if parenttype.startswith("`tab"):
-			# `tabChild DocType`.`fieldname`
-			parenttype = parenttype[4:-1]
-			fieldname = fieldname.strip("`")
+		df = frappe.get_meta(data.doctype).get_field(parenttype)
+		if not df and parenttype.startswith("tab"):
+			# tabChild DocType.fieldname
+			parenttype = parenttype[3:]
 		else:
 			# tablefield.fieldname
-			parenttype = frappe.get_meta(data.doctype).get_field(parenttype).options
+			parenttype = df.options
 	else:
 		parenttype = data.doctype
 		fieldname = field.strip("`")

frappe/email/doctype/email_account/email_account_list.js

@@ -16,7 +16,7 @@ frappe.listview_settings["Email Account"] = {
 			return [__("Default Sending"), color, "default_outgoing,=,Yes"];
 		} else {
 			color = doc.enable_incoming ? "blue" : "gray";
-			return [__("Inbox"), color, "is_global,=,No|is_default=No"];
+			return [__("Inbox"), color, "default_outgoing,=,No|default_incoming=No"];
 		}
 	},
 };