wcrisman/TempestDataCollection
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed bugs; Removed caching that would cause problems with spinning up multiple processes.

Browse Source
This commit is contained in:
DESKTOP-C9V2M01\Zeeri
2022-06-04 08:27:42 -07:00
parent 6287ab13cb
commit 92986cfb23
2 changed files with 38 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app.js
View File

@@ -10,12 +10,14 @@ var pingRouter = require('./routes/ping');
var app = express(); var app = express();
let port = 3003; let port = 3003;
let log = false;
if(process.env.PORT) port = process.env.PORT; if(process.env.PORT) port = process.env.PORT;
if(process.env.LOG) log = process.env.LOG;
console.log("Running on port: " + port); console.log("Running on port: " + port);
app.use(logger('dev')); if(log) app.use(logger('dev'));
app.use(express.json()); app.use(express.json());
app.use(express.urlencoded({ extended: false })); app.use(express.urlencoded({ extended: false }));
app.use(cookieParser()); app.use(cookieParser());

121
routes/ping.js
View File

@@ -13,7 +13,6 @@ if(!uri) {
} }
const client = new MongoClient(uri); const client = new MongoClient(uri);
let collection; let collection;
let records = {};
// We are keeping an in-memory set of current/active records for each Chromebook to reduce the number of reads without _id we perform. // We are keeping an in-memory set of current/active records for each Chromebook to reduce the number of reads without _id we perform.
// The idea is that each Chromebook pings every 5 minutes when a student is logged in and using the Chromebook. // The idea is that each Chromebook pings every 5 minutes when a student is logged in and using the Chromebook.
@@ -26,29 +25,16 @@ async function connect() {
await client.connect(); await client.connect();
const database = client.db("avusd_data_collection"); const database = client.db("avusd_data_collection");
collection = database.collection('records'); collection = database.collection('records');
console.log("Connected to Mongodb server"); // console.log("Connected to Mongodb server");
} catch(e) { } catch(e) {
console.log("error: caught ping.js:31 in Data Collection.");
console.log(e); console.log(e);
} }
} }
// Load a record for specific Chromebook by serial number (we assume they are unique).
async function loadRecord(serial) {
//Read record with given serial.
let record = await collection.findOne({serial, closed: false});
//If we found one then add it to the map.
if(record && !record.closed) {
records[record.serial] = record;
}
else {record = null;}
return record;
}
// Create a new record for a Chromebook & user. Set it in the cache (replace any existing one for that serial). // Create a new record for a Chromebook & user. Set it in the cache (replace any existing one for that serial).
async function createRecord(record, isInternal) { async function createRecord(record, isInternal) {
record.startTime = new Date().getTime(); record.startTime = record.endTime = new Date().getTime();
record.count = 1; record.count = 1;
record.internalCount = isInternal ? 1 : 0; record.internalCount = isInternal ? 1 : 0;
@@ -57,75 +43,35 @@ async function createRecord(record, isInternal) {
//Handle errors. //Handle errors.
if(queryResult.writeConcernError && queryResult.writeConcernError.errmsg) { if(queryResult.writeConcernError && queryResult.writeConcernError.errmsg) {
console.log("error: caught ping.js:61 in Data Collection.");
console.log(queryResult.writeConcernError.errmsg + " ::: " + JSON.stringify(record)); console.log(queryResult.writeConcernError.errmsg + " ::: " + JSON.stringify(record));
record = null;
} }
else {
//Add the map if there were no errors.
records[record.serial] = record;
}
return record;
} }
// Update a record. Will create a new record if one doesn't exist for the Chromebook. Will close a record if the user changes, and then create a new one for the new user.
async function updateRecord(record, isInternal, clientAddress) { async function updateRecord(record, isInternal, clientAddress) {
let existing = records[record.serial]; try {
if(!collection) {
if(!collection) { await connect();
await connect();
}
//If one is not in the cache, then read it from the db.
if(!existing) {
existing = await loadRecord(record.serial);
}
//If we couldn't find one for the given serial in the db, then create it.
// Note: We only do this if the connection is on the internal network.
// This prevents attackers from spamming the server with random id's to pollute the data.
if(!existing) {
if(isInternal) {
existing = await createRecord(record, isInternal);
} }
else {
//Log the external attempt to update a 'new' chromebook.
console.log("Ignoring external (" + clientAddress + ") input: " + JSON.stringify(record));
}
}
else {
let changed = {};
let inc = {};
//If the user has changed, then close the record. Create a new record for the new user. //Read record with given serial.
if(existing.email !== record.email) { let existing = await collection.findOne({serial: record.serial, closed: {$exists: false}});
changed.closed = existing.closed = true;
if(!existing) {
await createRecord(record, isInternal); await createRecord(record, isInternal);
} }
else { else {
//Update the time and count for both the db and the in memory data. //If the user has changed, then close the record. Create a new record for the new user.
changed.endTimestamp = existing.endTimestamp = new Date().getTime(); if(existing.email !== record.email) {
// Note: Cannot increment this way because there could be multiple instances of this process running with different cached values. Use MongoDB $inc instead. await collection.updateOne({_id: existing._id}, {$set: {closed: true}});
// changed.count = existing.count = existing.count + 1; await createRecord(record, isInternal);
inc.count = 1; }
else {
//Update the internal count if the Chromebook address is on the internal network. await collection.updateOne({_id: existing._id}, {$set: {endTime: new Date().getTime()}, $inc: {count: 1, internalCount: isInternal ? 1 : 0}});
if(isInternal) {
// Note: Cannot increment this way because there could be multiple instances of this process running with different cached values. Use MongoDB $inc instead.
// changed.internalCount = existing.internalCount = existing.internalCount + 1;
inc.internalCount = 1;
}
}
//If something changed, then update the db.
if(!_.isEmpty(changed)) {
let result = await collection.updateOne({_id: existing._id}, {$set: changed, $inc: inc});
//Error handling.
if(result.modifiedCount !== 1) {
console.log("Failed to update the record: " + JSON.stringify(existing));
} }
} }
} catch(e) {
console.log(e);
} }
} }
@@ -144,21 +90,24 @@ router.get('/', function(req, res, next) {
// console.log("Serial: " + params.serial); // console.log("Serial: " + params.serial);
// console.log("DeviceId: " + params.deviceId); // console.log("DeviceId: " + params.deviceId);
let clientAddress = req.header("X-Real-IP"); // Ignore any calls without a serial.
if(!clientAddress) clientAddress = req.socket.remoteAddress; if(params.serial) {
let isLocal = true; let clientAddress = req.header("X-Real-IP");
console.log("Found IP: " + clientAddress); if (!clientAddress) clientAddress = req.socket.remoteAddress;
let isLocal = true;
// console.log("Found IP: " + clientAddress);
if(localAddresses) { if (localAddresses) {
isLocal = false; isLocal = false;
for(let i = 0; !isLocal && i < localAddresses.length; i++) { for (let i = 0; !isLocal && i < localAddresses.length; i++) {
let next = localAddresses[i]; let next = localAddresses[i];
isLocal = clientAddress.startsWith(next); isLocal = clientAddress.startsWith(next);
}
} }
}
//Note: We are not waiting for this to finish. We don't care about the output. //Note: We are not waiting for this to finish. We don't care about the output.
updateRecord({serial, assetId, deviceId, email}, isLocal, clientAddress); updateRecord({serial, assetId, deviceId, email}, isLocal, clientAddress);
}
//Send response. Nothing for the moment. //Send response. Nothing for the moment.
res.setHeader("Access-Control-Allow-Origin", "*"); res.setHeader("Access-Control-Allow-Origin", "*");