Merge pull request #2641 from umami-software/analytics

v2.11.0

.gitignore

@@ -22,6 +22,7 @@ node_modules
 # misc
 .DS_Store
 .idea
+.yarn
 *.iml
 *.log
 .vscode

cypress.config.ts

@@ -4,4 +4,9 @@ export default defineConfig({
   e2e: {
     baseUrl: 'http://localhost:3000',
   },
+  // default username / password on init
+  env: {
+    umami_user: 'admin',
+    umami_password: 'umami',
+  },
 });

cypress/docker-compose.yml

@@ -43,9 +43,10 @@ services:
       - CYPRESS_umami_user=admin
       - CYPRESS_umami_password=umami
     volumes:
-      - ../tsconfig.json:/tsconfig.json
+      - ./tsconfig.json:/tsconfig.json
       - ../cypress.config.ts:/cypress.config.ts
       - ./:/cypress
       - ../node_modules/:/node_modules
+      - ../src/lib/crypto.ts:/src/lib/crypto.ts
 volumes:
   umami-db-data:

cypress/e2e/login.cy.ts

@@ -6,8 +6,12 @@ describe('Login tests', () => {
     },
     () => {
       cy.visit('/login');
-      cy.getDataTest('input-username').find('input').type(Cypress.env('umami_user'));
-      cy.getDataTest('input-password').find('input').type(Cypress.env('umami_password'));
+      cy.getDataTest('input-username').find('input').click();
+      cy.getDataTest('input-username').find('input').type(Cypress.env('umami_user'), { delay: 50 });
+      cy.getDataTest('input-password').find('input').click();
+      cy.getDataTest('input-password')
+        .find('input')
+        .type(Cypress.env('umami_password'), { delay: 50 });
       cy.getDataTest('button-submit').click();
       cy.url().should('eq', Cypress.config().baseUrl + '/dashboard');
       cy.getDataTest('button-profile').click();

cypress/e2e/website.cy.ts

@@ -10,8 +10,10 @@ describe('Website tests', () => {
     cy.visit('/settings/websites');
     cy.getDataTest('button-website-add').click();
     cy.contains(/Add website/i).should('be.visible');
-    cy.getDataTest('input-name').find('input').wait(500).type('Add test', { delay: 50 });
-    cy.getDataTest('input-domain').find('input').wait(500).type('addtest.com', { delay: 50 });
+    cy.getDataTest('input-name').find('input').click();
+    cy.getDataTest('input-name').find('input').type('Add test', { delay: 50 });
+    cy.getDataTest('input-domain').find('input').click();
+    cy.getDataTest('input-domain').find('input').type('addtest.com', { delay: 50 });
     cy.getDataTest('button-submit').click();
     cy.get('td[label="Name"]').should('contain.text', 'Add test');
     cy.get('td[label="Domain"]').should('contain.text', 'addtest.com');
@@ -26,27 +28,23 @@ describe('Website tests', () => {
         cy.deleteWebsite(websiteId);
       });
     cy.visit('/settings/websites');
-    cy.contains('Add test').should('not.exist');
+    cy.contains(/Add test/i).should('not.exist');
   });
 
-  it.only('Edit a website', () => {
+  it('Edit a website', () => {
     // prep data
     cy.addWebsite('Update test', 'updatetest.com');
     cy.visit('/settings/websites');
 
     // edit website
     cy.getDataTest('link-button-edit').first().click();
     cy.contains(/Details/i).should('be.visible');
-    cy.getDataTest('input-name')
-      .find('input')
-      .wait(500)
-      .clear()
-      .type('Updated website', { delay: 50 });
-    cy.getDataTest('input-domain')
-      .find('input')
-      .wait(500)
-      .clear()
-      .type('updatedwebsite.com', { delay: 50 });
+    cy.getDataTest('input-name').find('input').click();
+    cy.getDataTest('input-name').find('input').clear();
+    cy.getDataTest('input-name').find('input').type('Updated website', { delay: 50 });
+    cy.getDataTest('input-domain').find('input').click();
+    cy.getDataTest('input-domain').find('input').clear();
+    cy.getDataTest('input-domain').find('input').type('updatedwebsite.com', { delay: 50 });
     cy.getDataTest('button-submit').click({ force: true });
     cy.getDataTest('input-name').find('input').should('have.value', 'Updated website');
     cy.getDataTest('input-domain').find('input').should('have.value', 'updatedwebsite.com');
@@ -69,7 +67,7 @@ describe('Website tests', () => {
         cy.deleteWebsite(websiteId);
       });
     cy.visit('/settings/websites');
-    cy.contains('Add test').should('not.exist');
+    cy.contains(/Add test/i).should('not.exist');
   });
 
   it('Delete a website', () => {
@@ -86,6 +84,6 @@ describe('Website tests', () => {
     cy.contains(/Type DELETE in the box below to confirm./i).should('be.visible');
     cy.get('input[name="confirm"').type('DELETE');
     cy.get('button[type="submit"]').click();
-    cy.contains('Delete test').should('not.exist');
+    cy.contains(/Delete test/i).should('not.exist');
   });
 });

db/clickhouse/migrations/02_add_visit_id.sql

@@ -0,0 +1,90 @@
+CREATE TABLE umami.website_event_join
+(
+    session_id UUID,
+    visit_id UUID,
+    created_at DateTime('UTC')
+)
+    engine = MergeTree
+        ORDER BY (session_id, created_at)
+        SETTINGS index_granularity = 8192;
+
+INSERT INTO umami.website_event_join
+SELECT DISTINCT
+    s.session_id,
+    generateUUIDv4() visit_id,
+    s.created_at
+FROM (SELECT DISTINCT session_id,
+        date_trunc('hour', created_at) created_at
+    FROM website_event) s;
+
+-- create new table
+CREATE TABLE umami.website_event_new
+(
+    website_id UUID,
+    session_id UUID,
+    visit_id UUID,
+    event_id UUID,
+    hostname LowCardinality(String),
+    browser LowCardinality(String),
+    os LowCardinality(String),
+    device LowCardinality(String),
+    screen LowCardinality(String),
+    language LowCardinality(String),
+    country LowCardinality(String),
+    subdivision1 LowCardinality(String),
+    subdivision2 LowCardinality(String),
+    city String,
+    url_path String,
+    url_query String,
+    referrer_path String,
+    referrer_query String,
+    referrer_domain String,
+    page_title String,
+    event_type UInt32,
+    event_name String,
+    created_at DateTime('UTC'),
+    job_id UUID
+)
+    engine = MergeTree
+        ORDER BY (website_id, session_id, created_at)
+        SETTINGS index_granularity = 8192;
+
+INSERT INTO umami.website_event_new
+SELECT we.website_id,
+    we.session_id,
+    j.visit_id,
+    we.event_id,
+    we.hostname,
+    we.browser,
+    we.os,
+    we.device,
+    we.screen,
+    we.language,
+    we.country,
+    we.subdivision1,
+    we.subdivision2,
+    we.city,
+    we.url_path,
+    we.url_query,
+    we.referrer_path,
+    we.referrer_query,
+    we.referrer_domain,
+    we.page_title,
+    we.event_type,
+    we.event_name,
+    we.created_at,
+    we.job_id
+FROM umami.website_event we
+JOIN umami.website_event_join j
+    ON we.session_id = j.session_id
+        and date_trunc('hour', we.created_at) = j.created_at
+
+RENAME TABLE umami.website_event TO umami.website_event_old;
+RENAME TABLE umami.website_event_new TO umami.website_event;
+
+/*
+
+ DROP TABLE umami.website_event_old
+ DROP TABLE umami.website_event_join
+
+ */

db/clickhouse/schema.sql

@@ -3,6 +3,7 @@ CREATE TABLE umami.website_event
 (
     website_id UUID,
     session_id UUID,
+    visit_id UUID,
     event_id UUID,
     --sessions
     hostname LowCardinality(String),

db/mysql/migrations/05_add_visit_id/migration.sql

@@ -0,0 +1,22 @@
+-- AlterTable
+ALTER TABLE `website_event` ADD COLUMN `visit_id` VARCHAR(36) NULL;
+
+UPDATE `website_event` we
+JOIN (SELECT DISTINCT
+        s.session_id,
+        s.visit_time,
+        BIN_TO_UUID(RANDOM_BYTES(16) & 0xffffffffffff0fff3fffffffffffffff | 0x00000000000040008000000000000000) uuid
+    FROM (SELECT DISTINCT session_id,
+            DATE_FORMAT(created_at, '%Y-%m-%d %H:00:00') visit_time
+        FROM `website_event`) s) a
+    ON we.session_id = a.session_id and DATE_FORMAT(we.created_at, '%Y-%m-%d %H:00:00') = a.visit_time
+SET we.visit_id = a.uuid
+WHERE we.visit_id IS NULL;
+
+ALTER TABLE `website_event` MODIFY `visit_id` VARCHAR(36) NOT NULL;
+
+-- CreateIndex
+CREATE INDEX `website_event_visit_id_idx` ON `website_event`(`visit_id`);
+
+-- CreateIndex
+CREATE INDEX `website_event_website_id_visit_id_created_at_idx` ON `website_event`(`website_id`, `visit_id`, `created_at`);

db/mysql/schema.prisma

@@ -92,6 +92,7 @@ model WebsiteEvent {
   id             String    @id() @map("event_id") @db.VarChar(36)
   websiteId      String    @map("website_id") @db.VarChar(36)
   sessionId      String    @map("session_id") @db.VarChar(36)
+  visitId        String    @map("visit_id") @db.VarChar(36)
   createdAt      DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
   urlPath        String    @map("url_path") @db.VarChar(500)
   urlQuery       String?   @map("url_query") @db.VarChar(500)
@@ -107,6 +108,7 @@ model WebsiteEvent {
 
   @@index([createdAt])
   @@index([sessionId])
+  @@index([visitId])
   @@index([websiteId])
   @@index([websiteId, createdAt])
   @@index([websiteId, createdAt, urlPath])
@@ -115,6 +117,7 @@ model WebsiteEvent {
   @@index([websiteId, createdAt, pageTitle])
   @@index([websiteId, createdAt, eventName])
   @@index([websiteId, sessionId, createdAt])
+  @@index([websiteId, visitId, createdAt])
   @@map("website_event")
 }
 

db/postgresql/migrations/05_add_visit_id/migration.sql

@@ -0,0 +1,22 @@
+-- AlterTable
+ALTER TABLE "website_event" ADD COLUMN "visit_id" UUID NULL;
+
+UPDATE "website_event" we
+SET visit_id = a.uuid
+FROM (SELECT DISTINCT
+        s.session_id,
+        s.visit_time,
+        gen_random_uuid() uuid
+    FROM (SELECT DISTINCT session_id,
+            date_trunc('hour', created_at) visit_time
+        FROM "website_event") s) a
+WHERE we.session_id = a.session_id 
+    and date_trunc('hour', we.created_at) = a.visit_time;
+
+ALTER TABLE "website_event" ALTER COLUMN "visit_id" SET NOT NULL;
+
+-- CreateIndex
+CREATE INDEX "website_event_visit_id_idx" ON "website_event"("visit_id");
+
+-- CreateIndex
+CREATE INDEX "website_event_website_id_visit_id_created_at_idx" ON "website_event"("website_id", "visit_id", "created_at");

db/postgresql/schema.prisma

@@ -92,6 +92,7 @@ model WebsiteEvent {
   id             String    @id() @map("event_id") @db.Uuid
   websiteId      String    @map("website_id") @db.Uuid
   sessionId      String    @map("session_id") @db.Uuid
+  visitId        String    @map("visit_id") @db.Uuid
   createdAt      DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
   urlPath        String    @map("url_path") @db.VarChar(500)
   urlQuery       String?   @map("url_query") @db.VarChar(500)
@@ -107,6 +108,7 @@ model WebsiteEvent {
 
   @@index([createdAt])
   @@index([sessionId])
+  @@index([visitId])
   @@index([websiteId])
   @@index([websiteId, createdAt])
   @@index([websiteId, createdAt, urlPath])
@@ -115,6 +117,7 @@ model WebsiteEvent {
   @@index([websiteId, createdAt, pageTitle])
   @@index([websiteId, createdAt, eventName])
   @@index([websiteId, sessionId, createdAt])
+  @@index([websiteId, visitId, createdAt])
   @@map("website_event")
 }
 

next.config.js

@@ -14,6 +14,7 @@ const frameAncestors = process.env.ALLOWED_FRAME_URLS || '';
 const disableLogin = process.env.DISABLE_LOGIN || '';
 const disableUI = process.env.DISABLE_UI || '';
 const hostURL = process.env.HOST_URL || '';
+const privateMode = process.env.PRIVATE_MODE || '';
 
 const contentSecurityPolicy = [
   `default-src 'self'`,
@@ -120,6 +121,7 @@ const config = {
     disableLogin,
     disableUI,
     hostURL,
+    privateMode,
   },
   basePath,
   output: 'standalone',

package.json

@@ -1,6 +1,6 @@
 {
   "name": "umami",
-  "version": "2.10.2",
+  "version": "2.11.0",
   "description": "A simple, fast, privacy-focused alternative to Google Analytics.",
   "author": "Umami Software, Inc. <hello@umami.is>",
   "license": "MIT",
@@ -66,14 +66,14 @@
   "dependencies": {
     "@clickhouse/client": "^0.2.2",
     "@fontsource/inter": "^4.5.15",
-    "@prisma/client": "5.9.1",
+    "@prisma/client": "5.11.0",
     "@prisma/extension-read-replicas": "^0.3.0",
     "@react-spring/web": "^9.7.3",
-    "@tanstack/react-query": "^5.12.2",
+    "@tanstack/react-query": "^5.28.6",
     "@umami/prisma-client": "^0.14.0",
     "@umami/redis-client": "^0.18.0",
     "chalk": "^4.1.1",
-    "chart.js": "^4.2.1",
+    "chart.js": "^4.4.2",
     "chartjs-adapter-date-fns": "^3.0.0",
     "classnames": "^2.3.1",
     "colord": "^2.9.2",
@@ -98,11 +98,11 @@
     "maxmind": "^4.3.6",
     "md5": "^2.3.0",
     "moment-timezone": "^0.5.35",
-    "next": "14.1.0",
+    "next": "14.1.4",
     "next-basics": "^0.39.0",
     "node-fetch": "^3.2.8",
     "npm-run-all": "^4.1.5",
-    "prisma": "5.9.1",
+    "prisma": "5.11.0",
     "react": "^18.2.0",
     "react-basics": "^0.123.0",
     "react-beautiful-dnd": "^13.1.0",
@@ -115,7 +115,6 @@
     "request-ip": "^3.3.0",
     "semver": "^7.5.4",
     "thenby": "^1.3.4",
-    "timezone-support": "^2.0.2",
     "uuid": "^9.0.0",
     "yup": "^0.32.11",
     "zustand": "^4.3.8"
@@ -176,6 +175,6 @@
     "tar": "^6.1.2",
     "ts-jest": "^29.1.2",
     "ts-node": "^10.9.1",
-    "typescript": "^5.1.6"
+    "typescript": "^5.4.3"
   }
 }

public/intl/messages/am-ET.json

@@ -215,6 +215,12 @@
       "value": "Created"
     }
   ],
+  "label.created-by": [
+    {
+      "type": 0,
+      "value": "Created By"
+    }
+  ],
   "label.current-password": [
     {
       "type": 0,