Split streaming server from web server (#24702)

Procfile.dev

@@ -1,4 +1,4 @@
 web: env PORT=3000 RAILS_ENV=development bundle exec puma -C config/puma.rb
 sidekiq: env PORT=3000 RAILS_ENV=development bundle exec sidekiq
-stream: env PORT=4000 yarn run start
+stream: env PORT=4000 yarn workspace @mastodon/streaming start
 webpack: bin/webpack-dev-server

package.json

@@ -1,11 +1,13 @@
 {
   "name": "@mastodon/mastodon",
   "license": "AGPL-3.0-or-later",
+  "packageManager": "yarn@4.0.2",
   "engines": {
     "node": ">=18"
   },
   "workspaces": [
-    "."
+    ".",
+    "streaming"
   ],
   "scripts": {
     "build:development": "cross-env RAILS_ENV=development NODE_ENV=development ./bin/webpack",
@@ -71,10 +73,8 @@
     "css-loader": "^5.2.7",
     "cssnano": "^6.0.1",
     "detect-passive-events": "^2.0.3",
-    "dotenv": "^16.0.3",
     "emoji-mart": "npm:emoji-mart-lazyload@latest",
     "escape-html": "^1.0.3",
-    "express": "^4.18.2",
     "file-loader": "^6.2.0",
     "font-awesome": "^4.7.0",
     "fuzzysort": "^2.0.4",
@@ -85,21 +85,15 @@
     "immutable": "^4.3.0",
     "imports-loader": "^1.2.0",
     "intl-messageformat": "^10.3.5",
-    "ioredis": "^5.3.2",
     "js-yaml": "^4.1.0",
-    "jsdom": "^22.1.0",
     "lodash": "^4.17.21",
     "mark-loader": "^0.1.6",
     "marky": "^1.2.5",
     "mini-css-extract-plugin": "^1.6.2",
     "mkdirp": "^3.0.1",
-    "npmlog": "^7.0.1",
     "path-complete-extname": "^1.0.0",
-    "pg": "^8.5.0",
-    "pg-connection-string": "^2.6.0",
     "postcss": "^8.4.24",
     "postcss-loader": "^4.3.0",
-    "prom-client": "^15.0.0",
     "prop-types": "^15.8.1",
     "punycode": "^2.3.0",
     "react": "^18.2.0",
@@ -138,7 +132,6 @@
     "tesseract.js": "^2.1.5",
     "tiny-queue": "^0.2.1",
     "twitter-text": "3.1.0",
-    "uuid": "^9.0.0",
     "webpack": "^4.47.0",
     "webpack-assets-manifest": "^4.0.6",
     "webpack-bundle-analyzer": "^4.8.0",
@@ -150,8 +143,7 @@
     "workbox-routing": "^7.0.0",
     "workbox-strategies": "^7.0.0",
     "workbox-webpack-plugin": "^7.0.0",
-    "workbox-window": "^7.0.0",
-    "ws": "^8.12.1"
+    "workbox-window": "^7.0.0"
   },
   "devDependencies": {
     "@formatjs/cli": "^6.1.1",
@@ -160,16 +152,13 @@
     "@types/babel__core": "^7.20.1",
     "@types/emoji-mart": "^3.0.9",
     "@types/escape-html": "^1.0.2",
-    "@types/express": "^4.17.17",
     "@types/hoist-non-react-statics": "^3.3.1",
     "@types/http-link-header": "^1.0.3",
     "@types/intl": "^1.2.0",
     "@types/jest": "^29.5.2",
     "@types/js-yaml": "^4.0.5",
     "@types/lodash": "^4.14.195",
-    "@types/npmlog": "^4.1.4",
     "@types/object-assign": "^4.0.30",
-    "@types/pg": "^8.6.6",
     "@types/prop-types": "^15.7.5",
     "@types/punycode": "^2.1.0",
     "@types/react": "^18.2.7",
@@ -188,7 +177,6 @@
     "@types/react-toggle": "^4.0.3",
     "@types/redux-immutable": "^4.0.3",
     "@types/requestidlecallback": "^0.3.5",
-    "@types/uuid": "^9.0.0",
     "@types/webpack": "^4.41.33",
     "@types/yargs": "^17.0.24",
     "@typescript-eslint/eslint-plugin": "^6.0.0",
@@ -232,15 +220,10 @@
       "optional": true
     }
   },
-  "optionalDependencies": {
-    "bufferutil": "^4.0.7",
-    "utf-8-validate": "^6.0.3"
-  },
   "lint-staged": {
     "*": "prettier --ignore-unknown --write",
     "Capfile|Gemfile|*.{rb,ruby,ru,rake}": "bundle exec rubocop --force-exclusion -a",
     "*.{js,jsx,ts,tsx}": "eslint --fix",
     "*.{css,scss}": "stylelint --fix"
-  },
-  "packageManager": "yarn@4.0.2"
+  }
 }

streaming/index.js

@@ -2,6 +2,7 @@
 
 const fs = require('fs');
 const http = require('http');
+const path = require('path');
 const url = require('url');
 
 const dotenv = require('dotenv');
@@ -17,8 +18,11 @@ const WebSocket = require('ws');
 
 const environment = process.env.NODE_ENV || 'development';
 
+// Correctly detect and load .env or .env.production file based on environment:
+const dotenvFile = environment === 'production' ? '.env.production' : '.env';
+
 dotenv.config({
-  path: environment === 'production' ? '.env.production' : '.env',
+  path: path.resolve(__dirname, path.join('..', dotenvFile))
 });
 
 log.level = process.env.LOG_LEVEL || 'verbose';

streaming/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@mastodon/streaming",
+  "license": "AGPL-3.0-or-later",
+  "packageManager": "yarn@4.0.1",
+  "engines": {
+    "node": ">=18"
+  },
+  "description": "Mastodon's Streaming Server",
+  "private": true,
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/mastodon/mastodon.git"
+  },
+  "scripts": {
+    "start": "node ./index.js"
+  },
+  "dependencies": {
+    "dotenv": "^16.0.3",
+    "express": "^4.18.2",
+    "ioredis": "^5.3.2",
+    "jsdom": "^22.1.0",
+    "npmlog": "^7.0.1",
+    "pg": "^8.5.0",
+    "pg-connection-string": "^2.6.0",
+    "prom-client": "^15.0.0",
+    "uuid": "^9.0.0",
+    "ws": "^8.12.1"
+  },
+  "devDependencies": {
+    "@types/express": "^4.17.17",
+    "@types/npmlog": "^4.1.4",
+    "@types/pg": "^8.6.6",
+    "@types/uuid": "^9.0.0"
+  },
+  "optionalDependencies": {
+    "bufferutil": "^4.0.7",
+    "utf-8-validate": "^6.0.3"
+  }
+}