Tempest/imports/api/students.js

import { Meteor } from 'meteor/meteor';
import { Mongo } from 'meteor/mongo';
import { check } from 'meteor/check';
import {Sites} from "./sites";
import { Roles } from 'meteor/alanning:roles';
import {parse} from 'csv-parse';

export const Students = new Mongo.Collection('students');

if (Meteor.isServer) {
	Students.createIndex({id: 1}, {name: "External ID", unique: true});

	// This code only runs on the server
	Meteor.publish('students', function(siteId) {
		if(siteId) check(siteId, String);
		return siteId ? Students.find({siteId}) : Students.find({});
	});

	Meteor.methods({
		'students.add'(id, firstName, firstNameAlias, lastName, email, siteId, grade, active) {
			if(Roles.userIsInRole(Meteor.userId(), "admin", {anyScope:true})) {
				Students.insert({id, firstName, firstNameAlias, lastName, email, siteId, grade, active, activeChangeTimestamp: active ? "" : new Date()});
			}
		},
		'students.update'(_id, id, firstName, firstNameAlias, lastName, email, siteId, grade, active) {
			if(Roles.userIsInRole(Meteor.userId(), "admin", {anyScope:true})) {
				Students.update({_id}, {$set: {id, firstName, firstNameAlias, lastName, email, siteId, grade, active, activeChangeTimestamp: active ? "" : new Date()}});
			}
		},
		'students.remove'(_id) {
			// Does not actually remove the student (not currently possible.  Does set the student to not-active.
			// If we want to remove students we should allow it for non-active students if there are no assets assigned.
			// We may want to do this automatically, perhaps for students that have been non-active for a long period of time.
			if(Roles.userIsInRole(Meteor.userId(), "admin", {anyScope:true})) {
				// Set the student as non-active and set the timestamp for the change (so we know how long they have been inactive for - so we can potentially automatically remove them later.
				Students.update({_id}, {$set: {active: false, activeChangeTimestamp: new Date()}})
			}
		},
		'students.getPossibleGrades'() {
			return Students.rawCollection().distinct('grade', {});
		},
		/**
		 * Sets a first name alias that can be overridden by the one that is imported.
		 * @param _id The student's database ID.
		 * @param alias The alias to set for the student.
		 */
		'students.setAlias'(_id, alias) {
			if(Roles.userIsInRole(Meteor.userId(), "admin", {anyScope:true})) {
				check(_id, String);
				check(alias, String);

				Students.update({_id}, !alias || !alias.length() ? {$unset: {alias: true}} : {$set: {alias}});
			}
		},
		/**
		 * Assumes that the ID field is a unique ID that never changes for a student.
		 * This must be true in order for duplicate students to be avoided.
		 * Will automatically update a student's data, including the site he/she is associated with.
		 *
		 * Expects the CSV string to contain comma delimited data in the form:
		 * email, student ID, first name, last name, grade, first name alias, last name alias
		 *
		 * The query in Aeries is: `LIST STU NS ID SEM FN LN NG FNA IF NG <= 12`.
		 * A more complete Aeries query (for grades 7-12 in school 5): `LIST STU STU.NS STU.ID STU.SEM STU.FN STU.LN STU.NG BY STU.NG STU.SEM IF STU.NG >= 7 AND NG <= 12 AND STU.NS = 5`
		 * Note that FNA (First Name Alias) is optional.
		 * Note that you might want to include a school ID in the IF if you have multiple schools in the district.
		 * The query in SQL is: `SELECT [STU].[NS] AS [Next Schl], [STU].[ID] AS [Student ID], [STU].[SEM] AS [StuEmail], STU.FN AS [First Name], STU.LN AS [Last Name], [STU].[GR] AS [Grade], [STU].[FNA] AS [First Name Alias], [STU].[LNA] AS [Last Name Alias] FROM (SELECT [STU].* FROM STU WHERE [STU].DEL = 0) STU WHERE ( [STU].SC = 5) ORDER BY [STU].[LN], [STU].[FN];`.
		 * Run the query in Aeries as a `Report`, select TXT, and upload here.
		 * 
		 * Note: The headers for the CSV are not important and will be ignored.  The order of the data is very important.
		 *
		 * Aeries adds a header per 'page' of data (I think 35 entries per page).
		 * Example:
		 * Anderson Valley Jr/Sr High School,6/11/2022
		 * 2021-2022,Page 1
		 * Student ID, Email, First Name,Last Name,Grade,(opt) First Name Alias
		 * @type: Currently only supports 'csv' or 'aeries-txt'
		 * 
		 * TODO: We are assuming that we are importing all active students from the external system.  Any other assumption would require too much in the way of GUI
		 * TODO: Import should have a site id column that is the external site id.
		 * TODO: Each imported student should be attached to the correct site
		 * TODO: Any students not imported should be marked as deactivated
		 */
		'students.loadCsv'(csv, type, test) {
			try {
				if (Roles.userIsInRole(Meteor.userId(), "admin", {anyScope: true})) {
					check(csv, String)

					let sites = Sites.find().fetch()
					let sitesByExternalId = {}

					// Map all sites by external ID so we can quickly find the site for each imported student.
					for (let site of sites)
						if (site.externalId)
							sitesByExternalId[site.externalId] = site

					//Note: Only include active students since we don't want to repeatedly make students non-active (resetting the timestamp).
					let existingStudents = Students.find({active: true}).fetch()
					let existingStudentIds = new Set()

					// Collect all pre-existing student ID's.  Will remove them as we import, and use the remaining set to de-activate the students no longer in the district.
					for (let student of existingStudents)
						existingStudentIds.add(student.id)

					let cleanCsv
					let lines = csv.split(/\r?\n/)
					let pageHeader = type === 'aeries-txt' ? lines[0] : null  // Skip the repeating header lines for an Aeries text report.
					let skip = type === 'csv' ? 1 : 0  // Skip the first line of a CSV file (headers).

					// Remove headers from the CSV.
					for (const line of lines) {
						if (skip > 0) skip--
						else if (pageHeader && line === pageHeader) {
							skip = 2
						} else {
							if (!cleanCsv) cleanCsv = ""
							else cleanCsv += '\r\n'
							cleanCsv += line
						}
					}

					const bound = Meteor.bindEnvironment((callback) => {
						callback();
					})

					parse(cleanCsv, {}, function (err, records) {
						bound(() => {
							readCsv(err, records, sitesByExternalId, existingStudentIds, test)
						})
					})
				}
			} catch(err) {
				console.log(err)
			}
		}
	})

	/**
	 * Reads the CSV file containing Student data and updates and adds students to the system.  Students not in the CSV are marked as non-active.
	 * @param err
	 * @param records
	 * @param sitesByExternalId
	 * @param existingStudentIds
	 * @param test
	 * @returns {string}
	 */
	const readCsv = (err, records, sitesByExternalId, existingStudentIds, test) => {
		let output = ""
		
		if (err) console.error(err)
		else {
			let foundIds = new Set()
			let duplicates = []
			let count = 0
			let nonActiveCount = 0

			if (test)
				output += "Found " + records.length + " records.\r\n"

			try {
				for(const values of records) {
					let nextSchool = values[0]
					let siteId = sitesByExternalId[nextSchool] ? sitesByExternalId[nextSchool]._id : null
					let id = values[1]
					let email = values[2]
					let firstName = values[3]
					let lastName = values[4]
					let grade = parseInt(values[5], 10)
					let firstNameAlias = ""
					let active = true

					if (values.length > 6) firstNameAlias = values[6];

					// Ignore students at a site not in the system.
					if(siteId) {
						let student = { siteId, email, id, firstName, lastName, grade, firstNameAlias, active, activeChangeTimestamp: ""}

						// Track the student ID's and record duplicates.  This is used to ensure our counts are accurate later.
						// Note: We should never have duplicates in a perfect system, but in reality we do seem to end up with some duplicates in the SIS system's data.
						// There can be perfectly understandable reasons for this, so we will ignore them here since it shouldn't affect us.
						if (foundIds.has(student.id)) {
							duplicates.push(student.id)
						} else {
							foundIds.add(student.id)
						}

						count++

						if (!test) {
							try {
								existingStudentIds.delete(student.id)
								Students.upsert({id: student.id}, {$set: student})
							} catch (err) {
								console.log("Error while calling Students.upsert(..)")
								console.error(err)
							}
						} else {
							if (existingStudentIds.has(student.id)) {
								existingStudentIds.delete(student.id)
								output += "Updating existing student: " + student + "\r\n"
							} else output += "Adding student: " + student + "\r\n"
						}
					}
				}
			} catch(err) {
				console.log("Caught exception (while processing students imported via CSV): ")
				console.log(err)
			}

			// Change active status for all remaining students in the set (ones who were not in the import).
			for (let studentId of existingStudentIds) {
				nonActiveCount++

				if (test) {
					output += "Changing active status for student: " + Students.findOne({id: studentId}) + "\r\n"
				} else {
					try {
						Students.update({id: studentId}, {
							$set: {
								active: false,
								activeChangeTimestamp: new Date()
							}
						})
					} catch (err) {
						console.log("Student ID: " + studentId)
						console.log("Error updating Student to be non-active:")
						console.log(err)
					}
				}
			}

			console.log(duplicates.length + " records were duplicates:")
			console.log(duplicates)
			console.log("")
			console.log("Added or updated " + count + " students.")
			console.log("Update " + nonActiveCount + " students to non-active status.")
		}
		
		return output
	}

}