wcrisman/TempestDataCollection
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

First working version. Fixed problem with incrementing & cached data / multiple processes.

Browse Source
This commit is contained in:
DESKTOP-C9V2M01\Zeeri
2022-03-09 08:25:58 -08:00
commit 6287ab13cb
8 changed files with 5683 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
/node_modules/
/bin/
/.idea/
AvusdDataCollection.zip

35
app.js Normal file
View File

@@ -0,0 +1,35 @@
var express = require('express');
var path = require('path');
var cookieParser = require('cookie-parser');
var logger = require('morgan');
var sassMiddleware = require('node-sass-middleware');
var indexRouter = require('./routes/index');
var pingRouter = require('./routes/ping');
var app = express();
let port = 3003;
if(process.env.PORT) port = process.env.PORT;
console.log("Running on port: " + port);
app.use(logger('dev'));
app.use(express.json());
app.use(express.urlencoded({ extended: false }));
app.use(cookieParser());
app.use(sassMiddleware({
src: path.join(__dirname, 'public'),
dest: path.join(__dirname, 'public'),
indentedSyntax: true, // true = .sass and false = .scss
sourceMap: true
}));
//app.use(express.static(path.join(__dirname, 'public')));
app.use('/', indexRouter);
app.use('/ping', pingRouter);
app.listen(port);
module.exports = app;

5424
package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

21
package.json Normal file
View File

@@ -0,0 +1,21 @@
{
"name": "avusddatacollection",
"version": "0.0.0",
"private": true,
"scripts": {
"start": "node ./bin/www",
"build": "bestzip AvusdDataCollection.zip app.js bin public routes package*.json"
},
"dependencies": {
"cookie-parser": "~1.4.4",
"debug": "~2.6.9",
"express": "~4.16.1",
"mongodb": "^4.3.1",
"morgan": "~1.9.1",
"node-sass-middleware": "0.11.0",
"underscore": "^1.13.2"
},
"devDependencies": {
"bestzip": "^2.1.7"
}
}

13
public/index.html Normal file
View File

@@ -0,0 +1,13 @@
<html>
<head>
<title>Express</title>
<link rel="stylesheet" href="/stylesheets/style.css">
</head>
<body>
<h1>Express</h1>
<p>Welcome to Express</p>
</body>
</html>

6
public/stylesheets/style.sass Normal file
View File

@@ -0,0 +1,6 @@
body
padding: 50px
font: 14px "Lucida Grande", Helvetica, Arial, sans-serif
a
color: #00B7FF

10
routes/index.js Normal file
View File

@@ -0,0 +1,10 @@
var express = require('express');
var router = express.Router();
/* GET home page. */
router.get('/', function(req, res, next) {
//res.render('index', { title: 'Express' });
res.status(400);
});
module.exports = router;

170
routes/ping.js Normal file
View File

@@ -0,0 +1,170 @@
let _ = require("underscore");
let express = require('express');
let router = express.Router();
const {MongoClient} = require("mongodb");
let localAddresses = process.env.LOCAL_ADDRESSES; //eg: "10.18.,10.17."
// Create an array of local address beginnings. This is matched against the client addresses to determine if they are local. If not provided then all addresses are considered local.
if(localAddresses) localAddresses = localAddresses.split(',');
//const uri = "mongodb://test:1qaz2wsx@mongodb.avpanthers.org:27017/";
let uri = process.env.MONGO_URL; //Read from the nginx sites-available file for this app.
//mongodb://host1:27017,host2:27017,host3:27017/?replicaSet=myRs
if(!uri) {
uri = "mongodb://localhost/avusd_data_collection";
}
const client = new MongoClient(uri);
let collection;
let records = {};
// We are keeping an in-memory set of current/active records for each Chromebook to reduce the number of reads without _id we perform.
// The idea is that each Chromebook pings every 5 minutes when a student is logged in and using the Chromebook.
// We note how long between the first and last use of each chromebook & user, and how many pings were recorded during that time period.
// This gives us an idea of who used it for any given time period, and roughly how much use it got.
// Prepare a database connection for use later.
async function connect() {
try {
await client.connect();
const database = client.db("avusd_data_collection");
collection = database.collection('records');
console.log("Connected to Mongodb server");
} catch(e) {
console.log(e);
}
}
// Load a record for specific Chromebook by serial number (we assume they are unique).
async function loadRecord(serial) {
//Read record with given serial.
let record = await collection.findOne({serial, closed: false});
//If we found one then add it to the map.
if(record && !record.closed) {
records[record.serial] = record;
}
else {record = null;}
return record;
}
// Create a new record for a Chromebook & user. Set it in the cache (replace any existing one for that serial).
async function createRecord(record, isInternal) {
record.startTime = new Date().getTime();
record.count = 1;
record.internalCount = isInternal ? 1 : 0;
//Write to db.
let queryResult = await collection.insertOne(record);
//Handle errors.
if(queryResult.writeConcernError && queryResult.writeConcernError.errmsg) {
console.log(queryResult.writeConcernError.errmsg + " ::: " + JSON.stringify(record));
record = null;
}
else {
//Add the map if there were no errors.
records[record.serial] = record;
}
return record;
}
// Update a record. Will create a new record if one doesn't exist for the Chromebook. Will close a record if the user changes, and then create a new one for the new user.
async function updateRecord(record, isInternal, clientAddress) {
let existing = records[record.serial];
if(!collection) {
await connect();
}
//If one is not in the cache, then read it from the db.
if(!existing) {
existing = await loadRecord(record.serial);
}
//If we couldn't find one for the given serial in the db, then create it.
// Note: We only do this if the connection is on the internal network.
// This prevents attackers from spamming the server with random id's to pollute the data.
if(!existing) {
if(isInternal) {
existing = await createRecord(record, isInternal);
}
else {
//Log the external attempt to update a 'new' chromebook.
console.log("Ignoring external (" + clientAddress + ") input: " + JSON.stringify(record));
}
}
else {
let changed = {};
let inc = {};
//If the user has changed, then close the record. Create a new record for the new user.
if(existing.email !== record.email) {
changed.closed = existing.closed = true;
await createRecord(record, isInternal);
}
else {
//Update the time and count for both the db and the in memory data.
changed.endTimestamp = existing.endTimestamp = new Date().getTime();
// Note: Cannot increment this way because there could be multiple instances of this process running with different cached values. Use MongoDB $inc instead.
// changed.count = existing.count = existing.count + 1;
inc.count = 1;
//Update the internal count if the Chromebook address is on the internal network.
if(isInternal) {
// Note: Cannot increment this way because there could be multiple instances of this process running with different cached values. Use MongoDB $inc instead.
// changed.internalCount = existing.internalCount = existing.internalCount + 1;
inc.internalCount = 1;
}
}
//If something changed, then update the db.
if(!_.isEmpty(changed)) {
let result = await collection.updateOne({_id: existing._id}, {$set: changed, $inc: inc});
//Error handling.
if(result.modifiedCount !== 1) {
console.log("Failed to update the record: " + JSON.stringify(existing));
}
}
}
}
// Handle the client calling Ping.
router.get('/', function(req, res, next) {
//Get the parameters.
const params = req.query;
const email = params.email;
const assetId = params.assetId;
const serial = params.serial;
const deviceId = params.deviceId;
//DEBUG
// console.log("Email: " + params.email);
// console.log("AssetId: " + params.assetId);
// console.log("Serial: " + params.serial);
// console.log("DeviceId: " + params.deviceId);
let clientAddress = req.header("X-Real-IP");
if(!clientAddress) clientAddress = req.socket.remoteAddress;
let isLocal = true;
console.log("Found IP: " + clientAddress);
if(localAddresses) {
isLocal = false;
for(let i = 0; !isLocal && i < localAddresses.length; i++) {
let next = localAddresses[i];
isLocal = clientAddress.startsWith(next);
}
}
//Note: We are not waiting for this to finish. We don't care about the output.
updateRecord({serial, assetId, deviceId, email}, isLocal, clientAddress);
//Send response. Nothing for the moment.
res.setHeader("Access-Control-Allow-Origin", "*");
//res.setHeader("Content-Type", "application/json;charset=UTF8");
res.status(200);
res.end('Pong');
});
module.exports = router;