First working version. Fixed problem with incrementing & cached data / multiple processes.

2022-03-09 08:25:58 -08:00
commit 6287ab13cb
8 changed files with 5683 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -0,0 +1,4 @@
+/node_modules/
+/bin/
+/.idea/
+AvusdDataCollection.zip
--- a/app.js
+++ b/app.js
@@ -0,0 +1,35 @@
+var express = require('express');
+var path = require('path');
+var cookieParser = require('cookie-parser');
+var logger = require('morgan');
+var sassMiddleware = require('node-sass-middleware');
+
+var indexRouter = require('./routes/index');
+var pingRouter = require('./routes/ping');
+
+var app = express();
+
+let port = 3003;
+
+if(process.env.PORT) port = process.env.PORT;
+
+console.log("Running on port: " + port);
+
+app.use(logger('dev'));
+app.use(express.json());
+app.use(express.urlencoded({ extended: false }));
+app.use(cookieParser());
+app.use(sassMiddleware({
+  src: path.join(__dirname, 'public'),
+  dest: path.join(__dirname, 'public'),
+  indentedSyntax: true, // true = .sass and false = .scss
+  sourceMap: true
+}));
+//app.use(express.static(path.join(__dirname, 'public')));
+
+app.use('/', indexRouter);
+app.use('/ping', pingRouter);
+
+app.listen(port);
+
+module.exports = app;
--- a/package-lock.json
+++ b/package-lock.json
--- a/package.json
+++ b/package.json
@@ -0,0 +1,21 @@
+{
+  "name": "avusddatacollection",
+  "version": "0.0.0",
+  "private": true,
+  "scripts": {
+    "start": "node ./bin/www",
+    "build": "bestzip AvusdDataCollection.zip app.js bin public routes package*.json"
+  },
+  "dependencies": {
+    "cookie-parser": "~1.4.4",
+    "debug": "~2.6.9",
+    "express": "~4.16.1",
+    "mongodb": "^4.3.1",
+    "morgan": "~1.9.1",
+    "node-sass-middleware": "0.11.0",
+    "underscore": "^1.13.2"
+  },
+  "devDependencies": {
+    "bestzip": "^2.1.7"
+  }
+}
--- a/public/index.html
+++ b/public/index.html
@@ -0,0 +1,13 @@
+<html>
+
+<head>
+  <title>Express</title>
+  <link rel="stylesheet" href="/stylesheets/style.css">
+</head>
+
+<body>
+  <h1>Express</h1>
+  <p>Welcome to Express</p>
+</body>
+
+</html>
--- a/public/stylesheets/style.sass
+++ b/public/stylesheets/style.sass
@@ -0,0 +1,6 @@
+body
+  padding: 50px
+  font: 14px "Lucida Grande", Helvetica, Arial, sans-serif
+
+a
+  color: #00B7FF
--- a/routes/index.js
+++ b/routes/index.js
@@ -0,0 +1,10 @@
+var express = require('express');
+var router = express.Router();
+
+/* GET home page. */
+router.get('/', function(req, res, next) {
+  //res.render('index', { title: 'Express' });
+	res.status(400);
+});
+
+module.exports = router;
--- a/routes/ping.js
+++ b/routes/ping.js
@@ -0,0 +1,170 @@
+let _ = require("underscore");
+let express = require('express');
+let router = express.Router();
+const {MongoClient} = require("mongodb");
+let localAddresses = process.env.LOCAL_ADDRESSES;  //eg: "10.18.,10.17."
+// Create an array of local address beginnings.  This is matched against the client addresses to determine if they are local.  If not provided then all addresses are considered local.
+if(localAddresses) localAddresses = localAddresses.split(',');
+//const uri = "mongodb://test:1qaz2wsx@mongodb.avpanthers.org:27017/";
+let uri = process.env.MONGO_URL;  //Read from the nginx sites-available file for this app.
+//mongodb://host1:27017,host2:27017,host3:27017/?replicaSet=myRs
+if(!uri) {
+	uri = "mongodb://localhost/avusd_data_collection";
+}
+const client = new MongoClient(uri);
+let collection;
+let records = {};
+
+// We are keeping an in-memory set of current/active records for each Chromebook to reduce the number of reads without _id we perform.
+// The idea is that each Chromebook pings every 5 minutes when a student is logged in and using the Chromebook.
+// We note how long between the first and last use of each chromebook & user, and how many pings were recorded during that time period.
+// This gives us an idea of who used it for any given time period, and roughly how much use it got.
+
+// Prepare a database connection for use later.
+async function connect() {
+	try {
+		await client.connect();
+		const database = client.db("avusd_data_collection");
+		collection = database.collection('records');
+		console.log("Connected to Mongodb server");
+	} catch(e) {
+		console.log(e);
+	}
+}
+
+// Load a record for specific Chromebook by serial number (we assume they are unique).
+async function loadRecord(serial) {
+	//Read record with given serial.
+	let record = await collection.findOne({serial, closed: false});
+
+	//If we found one then add it to the map.
+	if(record && !record.closed) {
+		records[record.serial] = record;
+	}
+	else {record = null;}
+
+	return record;
+}
+
+// Create a new record for a Chromebook & user.  Set it in the cache (replace any existing one for that serial).
+async function createRecord(record, isInternal) {
+	record.startTime = new Date().getTime();
+	record.count = 1;
+	record.internalCount = isInternal ? 1 : 0;
+
+	//Write to db.
+	let queryResult = await collection.insertOne(record);
+
+	//Handle errors.
+	if(queryResult.writeConcernError && queryResult.writeConcernError.errmsg) {
+		console.log(queryResult.writeConcernError.errmsg + " ::: " + JSON.stringify(record));
+		record = null;
+	}
+	else {
+		//Add the map if there were no errors.
+		records[record.serial] = record;
+	}
+
+	return record;
+}
+
+// Update a record.  Will create a new record if one doesn't exist for the Chromebook.  Will close a record if the user changes, and then create a new one for the new user.
+async function updateRecord(record, isInternal, clientAddress) {
+	let existing = records[record.serial];
+
+	if(!collection) {
+		await connect();
+	}
+
+	//If one is not in the cache, then read it from the db.
+	if(!existing) {
+		existing = await loadRecord(record.serial);
+	}
+
+	//If we couldn't find one for the given serial in the db, then create it.
+	// Note: We only do this if the connection is on the internal network.
+	// This prevents attackers from spamming the server with random id's to pollute the data.
+	if(!existing) {
+		if(isInternal) {
+			existing = await createRecord(record, isInternal);
+		}
+		else {
+			//Log the external attempt to update a 'new' chromebook.
+			console.log("Ignoring external (" + clientAddress + ") input: " + JSON.stringify(record));
+		}
+	}
+	else {
+		let changed = {};
+		let inc = {};
+
+		//If the user has changed, then close the record.  Create a new record for the new user.
+		if(existing.email !== record.email) {
+			changed.closed = existing.closed = true;
+			await createRecord(record, isInternal);
+		}
+		else {
+			//Update the time and count for both the db and the in memory data.
+			changed.endTimestamp = existing.endTimestamp = new Date().getTime();
+			// Note: Cannot increment this way because there could be multiple instances of this process running with different cached values.  Use MongoDB $inc instead.
+			// changed.count = existing.count = existing.count + 1;
+			inc.count = 1;
+
+			//Update the internal count if the Chromebook address is on the internal network.
+			if(isInternal) {
+				// Note: Cannot increment this way because there could be multiple instances of this process running with different cached values.  Use MongoDB $inc instead.
+				// changed.internalCount = existing.internalCount = existing.internalCount + 1;
+				inc.internalCount = 1;
+			}
+		}
+
+		//If something changed, then update the db.
+		if(!_.isEmpty(changed)) {
+			let result = await collection.updateOne({_id: existing._id}, {$set: changed, $inc: inc});
+
+			//Error handling.
+			if(result.modifiedCount !== 1) {
+				console.log("Failed to update the record: " + JSON.stringify(existing));
+			}
+		}
+	}
+}
+
+// Handle the client calling Ping.
+router.get('/', function(req, res, next) {
+	//Get the parameters.
+	const params = req.query;
+	const email = params.email;
+	const assetId = params.assetId;
+	const serial = params.serial;
+	const deviceId = params.deviceId;
+
+	//DEBUG
+	// console.log("Email: " + params.email);
+	// console.log("AssetId: " + params.assetId);
+	// console.log("Serial: " + params.serial);
+	// console.log("DeviceId: " + params.deviceId);
+
+	let clientAddress = req.header("X-Real-IP");
+	if(!clientAddress) clientAddress = req.socket.remoteAddress;
+	let isLocal = true;
+	console.log("Found IP: " + clientAddress);
+
+	if(localAddresses) {
+		isLocal = false;
+		for(let i = 0; !isLocal && i < localAddresses.length; i++) {
+			let next = localAddresses[i];
+			isLocal = clientAddress.startsWith(next);
+		}
+	}
+
+	//Note: We are not waiting for this to finish.  We don't care about the output.
+	updateRecord({serial, assetId, deviceId, email}, isLocal, clientAddress);
+
+	//Send response.  Nothing for the moment.
+	res.setHeader("Access-Control-Allow-Origin", "*");
+	//res.setHeader("Content-Type", "application/json;charset=UTF8");
+	res.status(200);
+	res.end('Pong');
+});
+
+module.exports = router;