Merge branch 'development'

CHANGELOG.md

@@ -7,6 +7,7 @@ GRDB adheres to [Semantic Versioning](https://semver.org/), with one exception:
 
 #### 6.x Releases
 
+- `6.22.x` Releases - [6.22.0](#6220)
 - `6.21.x` Releases - [6.21.0](#6210)
 - `6.20.x` Releases - [6.20.0](#6200) - [6.20.1](#6201) - [6.20.2](#6202)
 - `6.19.x` Releases - [6.19.0](#6190)
@@ -119,6 +120,14 @@ GRDB adheres to [Semantic Versioning](https://semver.org/), with one exception:
 
 ---
 
+## 6.22.0
+
+Released November 26, 2023
+
+- **New**: [#1452](https://github.com/groue/GRDB.swift/pull/1452) by [@groue](https://github.com/groue): SQLite 3.44.0, FILTER and ORDER BY clauses in aggregate functions
+- **New**: [#1460](https://github.com/groue/GRDB.swift/pull/1460) by [@groue](https://github.com/groue): Explicit change notifications help applications deal with undetected database changes.
+- **Documentation Update**: The documentations of [`ValueObservation`](https://swiftpackageindex.com/groue/grdb.swift/documentation/grdb/valueobservation), [`DatabaseRegionObservation`](https://swiftpackageindex.com/groue/grdb.swift/documentation/grdb/databaseregionobservation), and [`TransactionObserver`](https://swiftpackageindex.com/groue/grdb.swift/documentation/grdb/transactionobserver) have a new "Dealing with Undetected Changes" that documents possible strategies for notifying applications of undetected database changes.  
+
 ## 6.21.0
 
 Released October 29, 2023

Documentation/CustomSQLiteBuilds.md

@@ -3,7 +3,7 @@ Custom SQLite Builds
 
 By default, GRDB uses the version of SQLite that ships with the target operating system.
 
-**You can build GRDB with a custom build of [SQLite 3.42.0](https://www.sqlite.org/changes.html).**
+**You can build GRDB with a custom build of [SQLite 3.44.0](https://www.sqlite.org/changes.html).**
 
 A custom SQLite build can activate extra SQLite features, and extra GRDB features as well, such as support for the [FTS5 full-text search engine](../../../#full-text-search), and [SQLite Pre-Update Hooks](https://swiftpackageindex.com/groue/grdb.swift/documentation/grdb/transactionobserver).
 

GRDB.swift.podspec

@@ -1,6 +1,6 @@
 Pod::Spec.new do |s|
   s.name     = 'GRDB.swift'
-  s.version  = '6.21.0'
+  s.version  = '6.22.0'
 
   s.license  = { :type => 'MIT', :file => 'LICENSE' }
   s.summary  = 'A toolkit for SQLite databases, with a focus on application development.'

GRDB/Core/Database+Schema.swift

@@ -57,6 +57,14 @@ extension Database {
             case let .attached(name): return "\(name).sqlite_master"
             }
         }
+
+        /// The name of the master sqlite table, without the schema name.
+        var unqualifiedMasterTableName: String { // swiftlint:disable:this inclusive_language
+            switch self {
+            case .main, .attached: return "sqlite_master"
+            case .temp: return "sqlite_temp_master"
+            }
+        }
     }
 
     /// The identifier of a database table or view.
@@ -658,9 +666,17 @@ extension Database {
     /// attached database.
     func canonicalTableName(_ tableName: String) throws -> String? {
         for schemaIdentifier in try schemaIdentifiers() {
+            // Regular tables
             if let result = try schema(schemaIdentifier).canonicalName(tableName, ofType: .table) {
                 return result
             }
+
+            // Master table (sqlite_master, sqlite_temp_master)
+            // swiftlint:disable:next inclusive_language
+            let masterTableName = schemaIdentifier.unqualifiedMasterTableName
+            if tableName.lowercased() == masterTableName.lowercased() {
+                return masterTableName
+            }
         }
         return nil
     }
@@ -1367,7 +1383,7 @@ struct SchemaObject: Hashable, FetchableRecord {
 
 /// All objects in a database schema (tables, views, indexes, triggers).
 struct SchemaInfo: Equatable {
-    private var objects: Set<SchemaObject>
+    let objects: Set<SchemaObject>
 
     /// Returns whether there exists a object of given type with this name
     /// (case-insensitive).

GRDB/Core/Database.swift

@@ -79,6 +79,7 @@ let SQLITE_TRANSIENT = unsafeBitCast(OpaquePointer(bitPattern: -1), to: sqlite3_
 /// - ``add(transactionObserver:extent:)``
 /// - ``remove(transactionObserver:)``
 /// - ``afterNextTransaction(onCommit:onRollback:)``
+/// - ``notifyChanges(in:)``
 /// - ``registerAccess(to:)``
 ///
 /// ### Collations
@@ -820,6 +821,65 @@ public final class Database: CustomStringConvertible, CustomDebugStringConvertib
         }
     }
 
+    /// Notifies that some changes were performed in the provided
+    /// database region.
+    ///
+    /// This method makes it possible to notify undetected changes, such as
+    /// changes performed by another process, changes performed by
+    /// direct calls to SQLite C functions, or changes to the
+    /// database schema.
+    /// See <doc:GRDB/TransactionObserver#Dealing-with-Undetected-Changes>
+    /// for a detailed list of undetected database modifications.
+    ///
+    /// It triggers active transaction observers (``TransactionObserver``).
+    /// In particular, ``ValueObservation`` that observe the input `region`
+    /// will fetch and notify a fresh value.
+    ///
+    /// For example:
+    ///
+    /// ```swift
+    /// try dbQueue.write { db in
+    ///     // Notify observers that some changes were performed in the database
+    ///     try db.notifyChanges(in: .fullDatabase)
+    ///
+    ///     // Notify observers that some changes were performed in the player table
+    ///     try db.notifyChanges(in: Player.all())
+    ///
+    ///     // Equivalent alternative
+    ///     try db.notifyChanges(in: Table("player"))
+    /// }
+    /// ```
+    ///
+    /// This method has no effect when called from a read-only
+    /// database access.
+    ///
+    /// > Caveat: Individual rowids in the input region are ignored.
+    /// > Notifying a change to a specific rowid is the same as notifying a
+    /// > change in the whole table:
+    /// >
+    /// > ```swift
+    /// > try dbQueue.write { db in
+    /// >     // Equivalent
+    /// >     try db.notifyChanges(in: Player.all())
+    /// >     try db.notifyChanges(in: Player.filter(id: 1))
+    /// > }
+    /// > ```
+    public func notifyChanges(in region: some DatabaseRegionConvertible) throws {
+        // Don't do anything when read-only, because read-only transactions
+        // are not notified. We don't want to notify transactions observers
+        // of changes, and have them wait for a commit notification that
+        // will never come.
+        if !isReadOnly, let observationBroker {
+            let eventKinds = try region
+                .databaseRegion(self)
+                // Use canonical table names for case insensitivity of the input.
+                .canonicalTables(self)
+                .impactfulEventKinds(self)
+
+            try observationBroker.notifyChanges(withEventsOfKind: eventKinds)
+        }
+    }
+
     /// Extends the `region` argument with the database region selected by all
     /// statements executed by the closure, and all regions explicitly tracked
     /// with the ``registerAccess(to:)`` method.

GRDB/Core/DatabaseFunction.swift

@@ -34,19 +34,6 @@ public final class DatabaseFunction: Hashable {
     private let kind: Kind
     private var eTextRep: CInt { (SQLITE_UTF8 | (isPure ? SQLITE_DETERMINISTIC : 0)) }
 
-    var functionFlags: SQLFunctionFlags {
-        var flags = SQLFunctionFlags(isPure: isPure)
-
-        switch kind {
-        case .function:
-            break
-        case .aggregate:
-            flags.isAggregate = true
-        }
-
-        return flags
-    }
-
     /// Creates an SQL function.
     ///
     /// For example:
@@ -157,6 +144,7 @@ public final class DatabaseFunction: Hashable {
         self.kind = .aggregate { Aggregate() }
     }
 
+    // TODO: GRDB7 -> expose ORDER BY and FILTER when we have distinct types for simple functions and aggregates.
     /// Returns an SQL expression that applies the function.
     ///
     /// You can use a `DatabaseFunction` as a regular Swift function. It returns
@@ -183,9 +171,24 @@ public final class DatabaseFunction: Hashable {
     /// }
     /// ```
     public func callAsFunction(_ arguments: any SQLExpressible...) -> SQLExpression {
-        .function(name, arguments.map(\.sqlExpression), flags: functionFlags)
+        switch kind {
+        case .function:
+            return .simpleFunction(
+                name,
+                arguments.map(\.sqlExpression),
+                isPure: isPure,
+                isJSONValue: false)
+        case .aggregate:
+            return .aggregateFunction(
+                name,
+                arguments.map(\.sqlExpression),
+                isDistinct: false,
+                ordering: nil,
+                filter: nil,
+                isJSONValue: false)
+        }
     }
-
+    
     /// Calls sqlite3_create_function_v2
     /// See <https://sqlite.org/c3ref/create_function.html>
     func install(in db: Database) {

GRDB/Core/DatabaseRegion.swift

@@ -169,16 +169,16 @@ public struct DatabaseRegion {
         // the observed region, we optimize database observation.
         //
         // And by canonicalizing table names, we remove views, and help the
-        // `isModified` methods.
+        // `isModified` methods. (TODO: is this comment still accurate?
+        // Isn't it about providing TransactionObserver.observes() with
+        // real tables names, instead?)
         try ignoringInternalSQLiteTables().canonicalTables(db)
     }
 
     /// Returns a region only made of actual tables with their canonical names.
-    /// Canonical names help the `isModified` methods.
     ///
-    /// This method removes views (assuming no table exists with the same name
-    /// as a view).
-    private func canonicalTables(_ db: Database) throws -> DatabaseRegion {
+    /// This method removes views.
+    func canonicalTables(_ db: Database) throws -> DatabaseRegion {
         guard let tableRegions else { return .fullDatabase }
         var region = DatabaseRegion()
         for (table, tableRegion) in tableRegions {
@@ -233,6 +233,44 @@ extension DatabaseRegion {
         }
         return tableRegion.contains(rowID: event.rowID)
     }
+
+    /// Returns an array of all event kinds that can impact this region.
+    ///
+    /// - precondition: the region is canonical.
+    func impactfulEventKinds(_ db: Database) throws -> [DatabaseEventKind] {
+        if let tableRegions {
+            return try tableRegions.flatMap { (table, tableRegion) -> [DatabaseEventKind] in
+                let tableName = table.rawValue // canonical table name
+                let columnNames: Set<String>
+                if let columns = tableRegion.columns {
+                    columnNames = Set(columns.map(\.rawValue))
+                } else {
+                    columnNames = try Set(db.columns(in: tableName).map(\.name))
+                }
+
+                return [
+                    DatabaseEventKind.delete(tableName: tableName),
+                    DatabaseEventKind.insert(tableName: tableName),
+                    DatabaseEventKind.update(tableName: tableName, columnNames: columnNames),
+                ]
+            }
+        } else {
+            // full database
+            return try db.schemaIdentifiers().flatMap { schemaIdentifier in
+                let schema = try db.schema(schemaIdentifier)
+                return try schema.objects
+                    .filter { $0.type == .table }
+                    .flatMap { table in
+                        let columnNames = try Set(db.columns(in: table.name).map(\.name))
+                        return [
+                            DatabaseEventKind.delete(tableName: table.name),
+                            DatabaseEventKind.insert(tableName: table.name),
+                            DatabaseEventKind.update(tableName: table.name, columnNames: columnNames),
+                        ]
+                    }
+            }
+        }
+    }
 }
 
 extension DatabaseRegion: Equatable {

GRDB/Core/DatabaseRegionObservation.swift

@@ -161,6 +161,11 @@ private class DatabaseRegionObserver: TransactionObserver {
         region.isModified(byEventsOfKind: eventKind)
     }
 
+    func databaseDidChange() {
+        isChanged = true
+        stopObservingDatabaseChangesUntilNextTransaction()
+    }
+
     func databaseDidChange(with event: DatabaseEvent) {
         if region.isModified(by: event) {
             isChanged = true

GRDB/Core/TransactionObserver.swift

@@ -282,6 +282,20 @@ class DatabaseObservationBroker {
         }
     }
 
+    func notifyChanges(withEventsOfKind eventKinds: [DatabaseEventKind]) throws {
+        // Support for stopObservingDatabaseChangesUntilNextTransaction()
+        SchedulingWatchdog.current!.databaseObservationBroker = self
+        defer {
+            SchedulingWatchdog.current!.databaseObservationBroker = nil
+        }
+
+        for observation in transactionObservations where observation.isEnabled {
+            if eventKinds.contains(where: { observation.observes(eventsOfKind: $0) }) {
+                observation.databaseDidChange()
+            }
+        }
+    }
+
     // MARK: - Statement execution
 
     /// Returns true if there exists some transaction observer interested in
@@ -566,6 +580,11 @@ class DatabaseObservationBroker {
         // even if we actually execute an empty deferred transaction.
         //
         // For better or for worse, let's simulate a transaction:
+        //
+        // 2023-11-26: I'm glad we did, because that's how we support calls
+        // to `Database.notifyChanges(in:)` from an empty transaction, as a
+        // way to tell transaction observers about changes performed by some
+        // external connection.
 
         do {
             try databaseWillCommit()
@@ -782,6 +801,16 @@ public protocol TransactionObserver: AnyObject {
     /// from being applied on the observed tables.
     func observes(eventsOfKind eventKind: DatabaseEventKind) -> Bool
 
+    /// Called when the database was modified in some unspecified way.
+    ///
+    /// This method allows a transaction observer to handle changes that are
+    /// not automatically detected. See <doc:GRDB/TransactionObserver#Dealing-with-Undetected-Changes>
+    /// and ``Database/notifyChanges(in:)`` for more information.
+    ///
+    /// The exact nature of changes is unknown, but they comply to the
+    /// ``observes(eventsOfKind:)`` test.
+    func databaseDidChange()
+
     /// Called when the database is changed by an insert, update, or
     /// delete event.
     ///
@@ -857,6 +886,9 @@ extension TransactionObserver {
     public func databaseWillChange(with event: DatabasePreUpdateEvent) { }
     #endif
 
+    /// The default implementation does nothing.
+    public func databaseDidChange() { }
+
     /// Prevents the observer from receiving further change notifications until
     /// the next transaction.
     ///
@@ -889,7 +921,7 @@ extension TransactionObserver {
         guard let broker = SchedulingWatchdog.current?.databaseObservationBroker else {
             fatalError("""
                 stopObservingDatabaseChangesUntilNextTransaction must be called \
-                from the databaseDidChange method
+                from the `databaseDidChange()` or `databaseDidChange(with:)` methods
                 """)
         }
         broker.disableUntilNextTransaction(transactionObserver: self)
@@ -942,6 +974,11 @@ final class TransactionObservation {
     }
     #endif
 
+    func databaseDidChange() {
+        guard isEnabled else { return }
+        observer?.databaseDidChange()
+    }
+
     func databaseDidChange(with event: DatabaseEvent) {
         guard isEnabled else { return }
         observer?.databaseDidChange(with: event)

GRDB/Documentation.docc/DatabaseSharing.md

@@ -228,9 +228,7 @@ In applications that use the background modes supported by iOS, post `resumeNoti
 
 <doc:DatabaseObservation> features are not able to detect database changes performed by other processes.
 
-Whenever you need to notify other processes that the database has been changed, you will have to use a cross-process notification mechanism such as [NSFileCoordinator] or [CFNotificationCenterGetDarwinNotifyCenter].
-
-You can trigger those notifications automatically with ``DatabaseRegionObservation``:
+Whenever you need to notify other processes that the database has been changed, you will have to use a cross-process notification mechanism such as [NSFileCoordinator] or [CFNotificationCenterGetDarwinNotifyCenter]. You can trigger those notifications automatically with ``DatabaseRegionObservation``:
 
 ```swift
 // Notify all changes made to the database
@@ -246,6 +244,8 @@ let observer = try observation.start(in: dbPool) { db in
 }
 ```
 
+The processes that observe the database can catch those notifications, and deal with the notified changes. See <doc:GRDB/TransactionObserver#Dealing-with-Undetected-Changes> for some related techniques.
+
 [NSFileCoordinator]: https://developer.apple.com/documentation/foundation/nsfilecoordinator
 [CFNotificationCenterGetDarwinNotifyCenter]: https://developer.apple.com/documentation/corefoundation/1542572-cfnotificationcentergetdarwinnot
 [WAL mode]: https://www.sqlite.org/wal.html