import { Meteor } from 'meteor/meteor'; import { Mongo } from 'meteor/mongo'; import { check } from 'meteor/check'; import {Sites} from "./sites"; import { Roles } from 'meteor/alanning:roles'; import {parse} from 'csv-parse'; export const Students = new Mongo.Collection('students'); if (Meteor.isServer) { await Students.createIndexAsync({id: 1}, {name: "External ID", unique: true}); // This code only runs on the server Meteor.publish('students', function(siteId) { if(siteId) check(siteId, String); return siteId ? Students.find({siteId}) : Students.find({}); }); Meteor.methods({ async 'students.add'(id, firstName, firstNameAlias, lastName, email, siteId, grade, active) { if(Roles.userIsInRole(Meteor.userId(), "admin", {anyScope:true})) { await Students.insertAsync({id, firstName, firstNameAlias, lastName, email, siteId, grade, active, activeChangeTimestamp: active ? "" : new Date()}); } }, async 'students.update'(_id, id, firstName, firstNameAlias, lastName, email, siteId, grade, active) { if(Roles.userIsInRole(Meteor.userId(), "admin", {anyScope:true})) { await Students.updateAsync({_id}, {$set: {id, firstName, firstNameAlias, lastName, email, siteId, grade, active, activeChangeTimestamp: active ? "" : new Date()}}); } }, async 'students.remove'(_id) { // Does not actually remove the student (not currently possible. Does set the student to not-active. // If we want to remove students we should allow it for non-active students if there are no assets assigned. // We may want to do this automatically, perhaps for students that have been non-active for a long period of time. if(Roles.userIsInRole(Meteor.userId(), "admin", {anyScope:true})) { // Set the student as non-active and set the timestamp for the change (so we know how long they have been inactive for - so we can potentially automatically remove them later. await Students.updateAsync({_id}, {$set: {active: false, activeChangeTimestamp: new Date()}}) } }, async 'students.getPossibleGrades'() { return Students.rawCollection().distinct('grade', {}); }, /** * Sets a first name alias that can be overridden by the one that is imported. * @param _id The student's database ID. * @param alias The alias to set for the student. */ async 'students.setAlias'(_id, alias) { if(Roles.userIsInRole(Meteor.userId(), "admin", {anyScope:true})) { check(_id, String); check(alias, String); await Students.updateAsync({_id}, !alias || !alias.length() ? {$unset: {alias: true}} : {$set: {alias}}); } }, /** * Assumes that the ID field is a unique ID that never changes for a student. * This must be true in order for duplicate students to be avoided. * Will automatically update a student's data, including the site he/she is associated with. * * Expects the CSV string to contain comma delimited data in the form: * email, student ID, first name, last name, grade, first name alias, last name alias * * The query in Aeries is: `LIST STU NS ID SEM FN LN NG FNA IF NG <= 12`. * A more complete Aeries query (for grades 7-12 in school 5): `LIST STU STU.NS STU.ID STU.SEM STU.FN STU.LN STU.NG BY STU.NG STU.SEM IF STU.NG >= 7 AND NG <= 12 AND STU.NS = 5` * Note that FNA (First Name Alias) is optional. * Note that you might want to include a school ID in the IF if you have multiple schools in the district. * The query in SQL is: `SELECT [STU].[NS] AS [Next Schl], [STU].[ID] AS [Student ID], [STU].[SEM] AS [StuEmail], STU.FN AS [First Name], STU.LN AS [Last Name], [STU].[GR] AS [Grade], [STU].[FNA] AS [First Name Alias], [STU].[LNA] AS [Last Name Alias] FROM (SELECT [STU].* FROM STU WHERE [STU].DEL = 0) STU WHERE ( [STU].SC = 5) ORDER BY [STU].[LN], [STU].[FN];`. * Run the query in Aeries as a `Report`, select TXT, and upload here. * * Note: The headers for the CSV are not important and will be ignored. The order of the data is very important. * * Aeries adds a header per 'page' of data (I think 35 entries per page). * Example: * Anderson Valley Jr/Sr High School,6/11/2022 * 2021-2022,Page 1 * Student ID, Email, First Name,Last Name,Grade,(opt) First Name Alias * @type: Currently only supports 'csv' or 'aeries-txt' * * TODO: We are assuming that we are importing all active students from the external system. Any other assumption would require too much in the way of GUI * TODO: Import should have a site id column that is the external site id. * TODO: Each imported student should be attached to the correct site * TODO: Any students not imported should be marked as deactivated */ async 'students.loadCsv'(csv, type, test) { try { if (Roles.userIsInRole(Meteor.userId(), "admin", {anyScope: true})) { check(csv, String) let sites = await Sites.find().fetchAsync() let sitesByExternalId = {} // Map all sites by external ID so we can quickly find the site for each imported student. for (let site of sites) if (site.externalId) sitesByExternalId[site.externalId] = site //Note: Only include active students since we don't want to repeatedly make students non-active (resetting the timestamp). let existingStudents = await Students.find({active: true}).fetchAsync() let existingStudentIds = new Set() // Collect all pre-existing student ID's. Will remove them as we import, and use the remaining set to de-activate the students no longer in the district. for (let student of existingStudents) existingStudentIds.add(student.id) let cleanCsv let lines = csv.split(/\r?\n/) let pageHeader = type === 'aeries-txt' ? lines[0] : null // Skip the repeating header lines for an Aeries text report. let skip = type === 'csv' ? 1 : 0 // Skip the first line of a CSV file (headers). // Remove headers from the CSV. for (const line of lines) { if (skip > 0) skip-- else if (pageHeader && line === pageHeader) { skip = 2 } else { if (!cleanCsv) cleanCsv = "" else cleanCsv += '\r\n' cleanCsv += line } } const bound = Meteor.bindEnvironment((callback) => { callback(); }) parse(cleanCsv, {}, function (err, records) { bound(() => { readCsv(err, records, sitesByExternalId, existingStudentIds, test) }) }) } } catch(err) { console.log(err) } } }) /** * Reads the CSV file containing Student data and updates and adds students to the system. Students not in the CSV are marked as non-active. * @param err * @param records * @param sitesByExternalId * @param existingStudentIds * @param test * @returns {string} */ const readCsv = async (err, records, sitesByExternalId, existingStudentIds, test) => { let output = "" if (err) console.error(err) else { let foundIds = new Set() let duplicates = [] let count = 0 let nonActiveCount = 0 if (test) output += "Found " + records.length + " records.\r\n" try { for(const values of records) { let nextSchool = values[0] let siteId = sitesByExternalId[nextSchool] ? sitesByExternalId[nextSchool]._id : null let id = values[1] let email = values[2] let firstName = values[3] let lastName = values[4] let grade = parseInt(values[5], 10) let firstNameAlias = "" let active = true if (values.length > 6) firstNameAlias = values[6]; // Ignore students at a site not in the system. if(siteId) { let student = { siteId, email, id, firstName, lastName, grade, firstNameAlias, active, activeChangeTimestamp: ""} // Track the student ID's and record duplicates. This is used to ensure our counts are accurate later. // Note: We should never have duplicates in a perfect system, but in reality we do seem to end up with some duplicates in the SIS system's data. // There can be perfectly understandable reasons for this, so we will ignore them here since it shouldn't affect us. if (foundIds.has(student.id)) { duplicates.push(student.id) } else { foundIds.add(student.id) } count++ if (!test) { try { existingStudentIds.delete(student.id) await Students.upsertAsync({id: student.id}, {$set: student}) } catch (err) { console.log("Error while calling Students.upsert(..)") console.error(err) } } else { if (existingStudentIds.has(student.id)) { existingStudentIds.delete(student.id) output += "Updating existing student: " + student + "\r\n" } else output += "Adding student: " + student + "\r\n" } } } } catch(err) { console.log("Caught exception (while processing students imported via CSV): ") console.log(err) } // Change active status for all remaining students in the set (ones who were not in the import). for (let studentId of existingStudentIds) { nonActiveCount++ if (test) { output += "Changing active status for student: " + await Students.findOneAsync({id: studentId}) + "\r\n" } else { try { await Students.updateAsync({id: studentId}, { $set: { active: false, activeChangeTimestamp: new Date() } }) } catch (err) { console.log("Student ID: " + studentId) console.log("Error updating Student to be non-active:") console.log(err) } } } console.log(duplicates.length + " records were duplicates:") console.log(duplicates) console.log("") console.log("Added or updated " + count + " students.") console.log("Update " + nonActiveCount + " students to non-active status.") } return output } }