Skip to content

Commit

Permalink
Fix slow place times with pixel index, cache board image on startup t…
Browse files Browse the repository at this point in the history 
…o eliminate delay
  • Loading branch information
@aydenp
aydenp committed Jun 23, 2018
1 parent abfd67f commit 88476fb
Show file tree
Hide file tree
Showing 7 changed files with 122 additions and 64 deletions.
3 changes: 2 additions & 1 deletion .gitignore
View file
Expand Up @@ -10,6 +10,7 @@ views/legit.*
config/community_guidelines.md
config/privacy_policy.md
config/tos.md
.place-data
modules/*

### Linux ###
Expand Down Expand Up @@ -75,4 +76,4 @@ $RECYCLE.BIN/

# Windows shortcuts
*.lnk
views/public/legit.js
views/public/legit.js
6 changes: 6 additions & 0 deletions app.js
View file
Expand Up @@ -15,6 +15,8 @@ const PixelNotificationManager = require("./util/PixelNotificationManager");
const JavaScriptProcessor = require("./util/JavaScriptProcessor");
const ChangelogManager = require("./util/ChangelogManager");
const User = require("./models/user");
const fs = require("fs");
const path = require("path");

var app = {};

Expand Down Expand Up @@ -53,6 +55,10 @@ app.reportError = app.logger.capture;
app.moduleManager = new ModuleManager(app);
app.moduleManager.loadAll();

// Create .place-data folder
app.dataFolder = path.resolve(__dirname, ".place-data");
if (!fs.existsSync(app.dataFolder)) fs.mkdirSync(app.dataFolder);

// Get image handler
app.paintingManager = PaintingManager(app);
app.logger.info('Startup', "Loading image from the database…");
Expand Down
15 changes: 10 additions & 5 deletions models/pixel.js
View file
Expand Up @@ -76,17 +76,21 @@ PixelSchema.statics.addPixel = function(colour, x, y, userID, app, callback) {
if(isNaN(x) || isNaN(y)) return callback(null, { message: "Invalid positions provided." });
// TODO: Get actual position below:
if(x < 0 || y < 0 || x >= app.config.boardSize || y >= app.config.boardSize) return callback(null, { message: "Position is out of bounds." });
this.find({
this.findOne({
xPos: x,
yPos: y
}).then((pixels) => {
}, {
editorID: 1,
colourR: 1,
colourG: 1,
colourB: 1
}).then((pixel) => {
// Find the pixel at this location
var pixel = pixels[0];
var wasIdentical = colour.r == 255 && colour.g == 255 && colour.b == 255; // set to identical if pixel was white
if(pixel) { // we have data from the old pixel
if (pixel) { // we have data from the old pixel
wasIdentical = pixel.editorID == userID && pixel.colourR == colour.r && pixel.colourG == colour.g && pixel.colourB == colour.b; // set to identical if colour matched old pixel
}
if(!wasIdentical) { // if the pixel was changed
if (!wasIdentical) { // if the pixel was changed
if(!pixel) { // if the spot was blank, create a new one
pixel = pn({
xPos: x,
Expand Down Expand Up @@ -140,4 +144,5 @@ PixelSchema.statics.getHexFromRGB = function(r, g, b) {
return componentToHex(r) + componentToHex(g) + componentToHex(b);
}

PixelSchema.index({xPos: 1, yPos: 1});
module.exports = DataModelManager.registerModel("Pixel", PixelSchema);
12 changes: 6 additions & 6 deletions models/user.js
View file
Expand Up @@ -303,13 +303,13 @@ UserSchema.methods.addPixel = function(colour, x, y, app, callback) {
if (changed) {
user.lastPlace = new Date();
user.placeCount++;
}
user.save(function(err) {
if (err) return callback(null, {
message: "An unknown error occurred while trying to place that pixel."
user.save((err) => {
if (err) return callback(null, {
message: "An unknown error occurred while trying to place that pixel."
});
return callback(changed, null);
});
return callback(changed, null);
})
} else callback(changed, null);
});
}

Expand Down
96 changes: 71 additions & 25 deletions util/PaintingManager.js
View file
@@ -1,9 +1,13 @@
const lwip = require("pajk-lwip");
const Pixel = require("../models/pixel");
const ActionLogger = require("../util/ActionLogger");
const fs = require("fs");
const path = require("path");

function PaintingManager(app) {
const imageSize = app.config.boardSize;
const cachedImagePath = path.resolve(app.dataFolder, "cached-board-image.png");
const temporaryCachedImagePath = path.resolve(app.dataFolder, "cached-board-image.png.tmp");
return {
hasImage: false,
imageHasChanged: false,
Expand All @@ -13,19 +17,49 @@ function PaintingManager(app) {
lastPixelUpdate: null,
firstGenerateAfterLoad: false,
pixelsToPaint: [],
pixelsToPreserve: null,
isGenerating: false,

getBlankImage: function() {
getStartingImage: function() {
return new Promise((resolve, reject) => {
lwip.create(imageSize, imageSize, "white", (err, image) => {
if (err) return reject(err);
resolve(image);
var resolveWithBlankImage = () => {
// Resolve with blank image if cache cannot be loaded
lwip.create(imageSize, imageSize, "white", (err, image) => {
if (err) return reject(err);
resolve({ image, canServe: false, skipImmediateCache: false });
});
}
// Check if cached image exists
fs.exists(cachedImagePath, (exists) => {
if (!exists) return resolveWithBlankImage();
// Attempt to load cached image
lwip.open(cachedImagePath, (err, image) => {
if (err) return resolveWithBlankImage();
if (image.width() != imageSize || image.height() != imageSize) return resolveWithBlankImage();
resolve({ image, canServe: true, skipImmediateCache: true });
})
});
});
},

loadImageFromDatabase: function() {
var hasServed = false;
return new Promise((resolve, reject) => {
this.getBlankImage().then((image) => {
var serveImage = async (image, skipImmediateCache = false) => {
this.hasImage = true;
this.image = image;
this.imageBatch = this.image.batch();
this.firstGenerateAfterLoad = true;
await this.generateOutputImage(skipImmediateCache);
if (!hasServed) resolve(image);
hasServed = true;
}
this.getStartingImage().then(async ({image, canServe, skipImmediateCache}) => {
if (canServe) {
app.logger.info("Startup", `Got initially serveable image, serving...`);
this.pixelsToPreserve = [];
await serveImage(image, skipImmediateCache);
}
let batch = image.batch();
Pixel.count({}).then((count) => {
var loaded = 0;
Expand All @@ -39,21 +73,20 @@ function PaintingManager(app) {
loaded++;
}).on("end", () => {
clearInterval(progressUpdater);
app.logger.info("Startup", `Loaded total ${count.toLocaleString()} pixel${count == 1 ? "" : "s"} pixels from database. Generating image...`);
app.logger.info("Startup", `Loaded total ${count.toLocaleString()} pixel${count == 1 ? "" : "s"} pixels from database. Applying to image...`);
if (this.pixelsToPreserve) this.pixelsToPreserve.forEach((data) => batch.setPixel(data.x, data.y, data.colour));
batch.exec((err, image) => {
this.pixelsToPreserve = null;
if (err) return reject(err);
this.hasImage = true;
this.image = image;
this.imageBatch = this.image.batch();
this.firstGenerateAfterLoad = true;
this.generateOutputImage();
resolve(image);
app.logger.info("Startup", `Applied pixels to image. Serving image...`);
serveImage(image);
});
}).on("error", (err) => {
this.pixelsToPreserve = null;
clearInterval(progressUpdater);
reject(err)
});
}).catch((err) => reject(err));
});
}).catch((err) => reject(err));
});
},
Expand All @@ -67,9 +100,11 @@ function PaintingManager(app) {
})
},

generateOutputImage: function() {
generateOutputImage: function(skipImmediateCache = false) {
var a = this;
return new Promise((resolve, reject) => {
if (a.isGenerating) return reject();
a.isGenerating = true;
this.waitingForImages.push((err, buffer) => {
if (err) return reject(err);
resolve(buffer);
Expand All @@ -83,10 +118,21 @@ function PaintingManager(app) {
this.pixelsToPaint = [];
this.imageBatch.toBuffer("png", { compression: "fast", transparency: false }, (err, buffer) => {
a.outputImage = buffer;
a.isGenerating = false;
if (!err && !skipImmediateCache) {
fs.writeFile(temporaryCachedImagePath, buffer, (err) => {
if (err) return app.logger.error("Painting Manager", "Couldn't save cached board image, got error:", err);
if (fs.existsSync(cachedImagePath)) fs.unlinkSync(cachedImagePath);
fs.rename(temporaryCachedImagePath, cachedImagePath, (err) => {
if (err) return app.logger.error("Painting Manager", "Couldn't move cached board image into place, got error:", err)
app.logger.info("Painting Manager", "Saved cached board image successfully!");
})
});
}
a.imageHasChanged = false;
a.waitingForImages.forEach((callback) => callback(err, buffer));
a.waitingForImages = [];
if(a.firstGenerateAfterLoad) {
if (a.firstGenerateAfterLoad) {
app.websocketServer.broadcast("server_ready");
a.firstGenerateAfterLoad = false;
}
Expand All @@ -107,14 +153,16 @@ function PaintingManager(app) {
doPaint: function(colour, x, y, user) {
var a = this;
return new Promise((resolve, reject) => {
if(!this.hasImage) return reject({message: "Our servers are currently getting ready. Please try again in a moment.", code: "not_ready"});
if(app.temporaryUserInfo.isUserPlacing(user)) return reject({message: "You cannot place more than one tile at once.", code: "attempted_overload"});
if (!this.hasImage) return reject({message: "Our servers are currently getting ready. Please try again in a moment.", code: "not_ready"});
if (app.temporaryUserInfo.isUserPlacing(user)) return reject({message: "You cannot place more than one tile at once.", code: "attempted_overload"});
app.temporaryUserInfo.setUserPlacing(user, true);
// Add to DB:
user.addPixel(colour, x, y, app, (changed, err) => {
app.temporaryUserInfo.setUserPlacing(user, false);
if(err) return reject(err);
a.pixelsToPaint.push({x: x, y: y, colour: colour});
if (err) return reject(err);
const pixelData = {x: x, y: y, colour: colour};
a.pixelsToPaint.push(pixelData);
if (a.pixelsToPreserve) a.pixelsToPreserve.push(pixelData);
a.imageHasChanged = true;
// Send notice to all clients:
var info = {x: x, y: y, colour: Pixel.getHexFromRGB(colour.r, colour.g, colour.b)};
Expand All @@ -129,12 +177,10 @@ function PaintingManager(app) {

startTimer: function() {
setInterval(() => {
if(this.imageHasChanged) {
app.logger.log('Painting Manager', "Starting board image update...");
this.generateOutputImage();
} else {
app.logger.log('Painting Manager', "Not updating board image, no changes since last update.");
}
if (this.pixelsToPreserve) return app.logger.log("Painting Manager", "Will not start board image update, as board image is still being completely loaded...");
if (!this.imageHasChanged) return app.logger.log("Painting Manager", "Not updating board image, no changes since last update.");
app.logger.log("Painting Manager", "Starting board image update...");
this.generateOutputImage();
}, 30 * 1000);
}
};
Expand Down
4 changes: 2 additions & 2 deletions views/layout.pug
View file
Expand Up @@ -10,7 +10,7 @@ block dependencies
- if(typeof isPoppedOut === "undefined") isPoppedOut = false;
- var userAdmin = user ? user.admin : false, userMod = user ? user.moderator || userAdmin : false;
- if(typeof needsHelp === "undefined") needsHelp = false;
- var resourceVersion = "8"; // Increment this when you make a change to the HTML that absolutely requires CSS or JS refreshing
- var resourceVersion = "9"; // Increment this when you make a change to the HTML that absolutely requires CSS or JS refreshing
- css = resources.css.concat(css);
- var bodyClasses = ["fixed-navbar"];
- if (user) bodyClasses.push("signed-in");
Expand Down Expand Up @@ -194,4 +194,4 @@ html(lang="en")

ga("create", "#{config.googleAnalyticsTrackingID}", "auto");
ga("send", "pageview");
});
});
50 changes: 25 additions & 25 deletions views/public/views/auth-dialog.pug
View file
Expand Up @@ -42,7 +42,7 @@
else
br
div.alert.alert-danger
h4 Signups for #{config.siteName} are currently disabled.
h4 Signing up for #{config.siteName} is currently disabled.
if config.maintenance.notice
p !{config.maintenance.notice}

Expand All @@ -51,28 +51,28 @@
span.site #{config.siteName}
h1 Welcome back
p.subhead Sign in to your account to continue placing and save your stats.
if config.maintenance.allowLogins || !config.maintenance
form.form-signin(action="/signin")
label.sr-only(for="inputUsername") Username
input.form-control#inputUsername(type="text", autocomplete="username", placeholder="Username", name="username", required, autofocus, autocorrect="off", autocapitalizae="off", spellcheck="false")
label.sr-only(for="inputPassword") Password
input.form-control#inputPassword(type="password", autocomplete="current-password", placeholder="Password", name="password", required)
.checkbox
label
input#inputKeepSignIn(type="checkbox", name="keepSignedIn", checked)
span Keep me signed in
.send-section
button.btn.btn-popping(type="submit") Sign in
include social-buttons
if config.signInBanner
br
div.alert.alert-success !{config.signInBanner}
else
br
div.alert.alert-danger
h4 Signins to #{config.siteName} are currently disabled.
if config.maintenance.notice
p !{config.maintenance.notice}
if config.maintenance.allowLogins || !config.maintenance
form.form-signin(action="/signin")
label.sr-only(for="inputUsername") Username
input.form-control#inputUsername(type="text", autocomplete="username", placeholder="Username", name="username", required, autofocus, autocorrect="off", autocapitalize="off", spellcheck="false")
label.sr-only(for="inputPassword") Password
input.form-control#inputPassword(type="password", autocomplete="current-password", placeholder="Password", name="password", required)
.checkbox
label
input#inputKeepSignIn(type="checkbox", name="keepSignedIn", checked)
span Keep me signed in
.send-section
button.btn.btn-popping(type="submit") Sign in
include social-buttons
if config.signInBanner
br
div.alert.alert-success !{config.signInBanner}
else
br
div.alert.alert-danger
h4 Signing in to #{config.siteName} is currently disabled.
if config.maintenance.notice
p !{config.maintenance.notice}
div.hides-switchers(tab-name="2fa-auth")
.heading
span.site #{config.siteName}
Expand All @@ -83,7 +83,7 @@
input.form-control#inputPassword2FA(type="hidden", name="password")
input#inputKeepSignIn2FA(type="checkbox", name="keepSignedIn", checked, style="display: none;")
label.sr-only(for="inputTotpToken") Authentication Token
input.form-control#inputTotpToken(maxlength="6", inputmode="numeric", pattern="[0-9]{6}", name="totpToken", placeholder="Authentication Token", required)
input.form-control#inputTotpToken(maxlength="6", inputmode="numeric", pattern="[0-9]{6}", name="totpToken", placeholder="Authentication Token", required, autocorrect="off", autocapitalize="off", autocomplete="off", spellcheck="false")
.send-section
button.btn.btn-popping(type="submit") Continue
.switchers
Expand All @@ -100,4 +100,4 @@
span.headline Already a member?
span.action-label Sign in to your account
.call-to-action
a.btn.btn-popping.btn-arrow Sign In
a.btn.btn-popping.btn-arrow Sign In

0 comments on commit 88476fb

Please sign in to comment.