Self host test

Dockerfile

@@ -0,0 +1,39 @@
+FROM node:16-alpine AS node
+FROM node AS node-with-gyp
+RUN apk add g++ make python3
+FROM node-with-gyp AS builder
+WORKDIR /squid
+ADD package.json .
+ADD package-lock.json .
+# remove if needed
+ADD assets assets 
+# remove if needed
+ADD db db
+# remove if needed
+ADD schema.graphql .
+RUN npm ci
+ADD tsconfig.json .
+ADD src src
+RUN npm run build
+FROM node-with-gyp AS deps
+WORKDIR /squid
+ADD package.json .
+ADD package-lock.json .
+RUN npm ci --production
+FROM node AS squid
+WORKDIR /squid
+COPY --from=deps /squid/package.json .
+COPY --from=deps /squid/package-lock.json .
+COPY --from=deps /squid/node_modules node_modules
+COPY --from=builder /squid/lib lib
+# remove if no assets folder
+COPY --from=builder /squid/assets assets
+# remove if no db folder
+COPY --from=builder /squid/db db
+# remove if no schema.graphql is in the root
+COPY --from=builder /squid/schema.graphql schema.graphql
+# remove if no commands.json is in the root
+ADD commands.json .
+RUN echo -e "loglevel=silent\\nupdate-notifier=false" > /squid/.npmrc
+RUN npm i -g @subsquid/commands && mv $(which squid-commands) /usr/local/bin/sqd
+ENV PROCESSOR_PROMETHEUS_PORT 3000

README.md

@@ -13,6 +13,7 @@ npm i -g @subsquid/cli
 ```
 
 ## Running locally
+### SQD
 ```bash
 # 1. Install dependencies
 npm ci
@@ -28,6 +29,12 @@ sqd serve
 
 # 5. Access via http://localhost:4350/graphql
 ```
+### DOCKER
+```bash
+# https://docs.subsquid.io/deploy-squid/self-hosting/
+docker build . -t my-squid
+docker compose up
+```
 
 ## Deployment
 

docker-compose.yml

@@ -2,11 +2,38 @@ version: "3"
 
 services:
   db:
-    image: postgres:12
+    image: postgres:15
     environment:
       POSTGRES_DB: squid
       POSTGRES_PASSWORD: postgres
     ports:
-      - "${DB_PORT}:5432"
+      - "5432:5432"
+      # Uncomment for logging all SQL statements
       # command: ["postgres", "-c", "log_statement=all"]
-    shm_size: 1gb
+  api:
+    image: my-squid
+    environment:
+      - DB_NAME=squid
+      - DB_PORT=5432
+      - DB_HOST=db
+      - DB_PASS=postgres
+      - GQL_PORT=4350
+    ports:
+      # GraphQL endpoint at port 4350
+      - "4350:4350"
+    command: ["sqd", "serve:prod"]
+    depends_on:
+      - db
+  processor:
+    image: my-squid
+    environment:
+      - DB_NAME=squid
+      - DB_PORT=5432
+      - DB_HOST=db
+      - DB_PASS=postgres
+    ports:
+      # prometheus metrics exposed at port 3000
+      - "3000:3000"
+    command: ["sqd", "process:prod"]
+    depends_on:
+      - db

src/processor.ts

@@ -61,37 +61,23 @@ processor.run(new TypeormDatabase(), async (ctx) => {
     );
   }
 
-  // 3. Create new group users
-  const groupUsers = await extractGroupUsers(ctx);
-  const newGroupUsers = groupUsers.map((groupUser) => {
-    return new GroupUser({
-      id: groupUser.id,
-      group: groupUser.group,
-      accountId: groupUser.accountId,
-      role: groupUser.role,
-    });
-  });
-  await ctx.store.insert(newGroupUsers);
-
-  // 4. Update group users
-  const groupUsersUpdate = await extractGroupUsersUpdate(ctx);
-  for (const groupUser of groupUsersUpdate) {
-    await ctx.store.save(
-      new GroupUser({
-        id: groupUser.id,
-        group: groupUser.group,
-        accountId: groupUser.accountId,
-        role: groupUser.role,
-      }),
-    );
+  // 3. Process group user events (needs to be done this way as group users can be destroyed)
+  const groupUserEvents = await extractGroupUserEvents(ctx);
+  for (const groupUserEvent of groupUserEvents) {
+    if (groupUserEvent.action == 'destroy') {
+      await ctx.store.remove(GroupUser, groupUserEvent.id);
+    } else {
+      await ctx.store.save(
+        new GroupUser({
+          id: groupUserEvent.id,
+          group: groupUserEvent.group,
+          accountId: groupUserEvent.accountId,
+          role: groupUserEvent.role,
+        }),
+      );
+    }
   };
 
-  // 5. Delete group users
-  const groupUsersDestroy = extractGroupUsersDestroy(ctx);
-  if (groupUsersDestroy.length) {
-    await ctx.store.remove(GroupUser, groupUsersDestroy);
-  }
-
   // 6. Create new smart contracts
   const newSmartContracts = smartContracts.map((smartContract) => {
     return new SmartContract({
@@ -142,6 +128,7 @@ interface GroupUserEvent {
   group: Group;
   accountId: string;
   role: string;
+  action: string;
 }
 
 interface SmartContractCreateEvent {
@@ -215,14 +202,15 @@ function extractGroupsUpdate(ctx: Ctx): GroupEvent[] {
   return groups;
 }
 
-async function extractGroupUsers(ctx: Ctx): Promise<GroupUserEvent[]> {
+async function extractGroupUserEvents(ctx: Ctx): Promise<GroupUserEvent[]> {
   const groupUsers: GroupUserEvent[] = [];
   for (const block of ctx.blocks) {
     for (const item of block.items) {
       if (
         item.name === "Contracts.ContractEmitted" &&
         item.event.args.contract === GROUPS_CONTRACT_ADDRESS
       ) {
+
         const event = azGroups.decodeEvent(item.event.args.data);
         if (event.__kind === "GroupUserCreate") {
           const group = await ctx.store.get(Group, String(event.groupId));
@@ -234,25 +222,10 @@ async function extractGroupUsers(ctx: Ctx): Promise<GroupUserEvent[]> {
               group,
               accountId: ss58.codec(SS58_PREFIX).encode(event.user),
               role: event.role.__kind,
+              action: "create"
             });
           }
-        }
-      }
-    }
-  }
-  return groupUsers;
-}
-
-async function extractGroupUsersUpdate(ctx: Ctx): Promise<GroupUserEvent[]> {
-  const groupUsers: GroupUserEvent[] = [];
-  for (const block of ctx.blocks) {
-    for (const item of block.items) {
-      if (
-        item.name === "Contracts.ContractEmitted" &&
-        item.event.args.contract === GROUPS_CONTRACT_ADDRESS
-      ) {
-        const event = azGroups.decodeEvent(item.event.args.data);
-        if (event.__kind === "GroupUserUpdate") {
+        } else if (event.__kind === "GroupUserUpdate") {
           const group = await ctx.store.get(Group, String(event.groupId));
           if (group) {
             groupUsers.push({
@@ -262,6 +235,20 @@ async function extractGroupUsersUpdate(ctx: Ctx): Promise<GroupUserEvent[]> {
               group,
               accountId: ss58.codec(SS58_PREFIX).encode(event.user),
               role: event.role.__kind,
+              action: "update"
+            });
+          }
+        } else if (event.__kind === "GroupUserDestroy") {
+          const group = await ctx.store.get(Group, String(event.groupId));
+          if (group) {
+            groupUsers.push({
+              id: `${event.groupId}-${ss58
+                .codec(SS58_PREFIX)
+                .encode(event.user)}`,
+              group,
+              accountId: ss58.codec(SS58_PREFIX).encode(event.user),
+              role: "destroy",
+              action: "destroy"
             });
           }
         }
@@ -271,26 +258,6 @@ async function extractGroupUsersUpdate(ctx: Ctx): Promise<GroupUserEvent[]> {
   return groupUsers;
 }
 
-function extractGroupUsersDestroy(ctx: Ctx): string[] {
-  const groupUserIds: string[] = [];
-  for (const block of ctx.blocks) {
-    for (const item of block.items) {
-      if (
-        item.name === "Contracts.ContractEmitted" &&
-        item.event.args.contract === GROUPS_CONTRACT_ADDRESS
-      ) {
-        const event = azGroups.decodeEvent(item.event.args.data);
-        if (event.__kind === "GroupUserDestroy") {
-          groupUserIds.push(
-            `${event.groupId}-${ss58.codec(SS58_PREFIX).encode(event.user)}`,
-          );
-        }
-      }
-    }
-  }
-  return groupUserIds;
-}
-
 async function extractSmartContracts(ctx: Ctx): Promise<SmartContractCreateEvent[]> {
   const smartContracts: SmartContractCreateEvent[] = [];
   for (const block of ctx.blocks) {