From e999beb8b6a9b793b622b04100ae5b4f76b895d7 Mon Sep 17 00:00:00 2001 From: Mars Ultor Date: Thu, 18 Dec 2025 17:23:24 -0600 Subject: [PATCH] thread is running now too, but no full workflow executed --- SkywardGradeChecker | 2 +- src/main.cpp | 66 ++++++++++- src/skwyward-api-utils.cpp | 44 +++++-- src/types.cpp | 228 ++++++++++++++++++++++++++----------- 4 files changed, 259 insertions(+), 81 deletions(-) diff --git a/SkywardGradeChecker b/SkywardGradeChecker index dbba005..d5f45db 160000 --- a/SkywardGradeChecker +++ b/SkywardGradeChecker @@ -1 +1 @@ -Subproject commit dbba00532754cf3d22ef70fbda5b62af8f7aa3d7 +Subproject commit d5f45dba8f70882b6d6f6923c1c97f0821e36941 diff --git a/src/main.cpp b/src/main.cpp index 76c0e23..756a2f7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,16 +1,60 @@ #include "clickhouse/client.h" +#include "cpr/api.h" +#include "cpr/response.h" #include "nlohmann/json_fwd.hpp" #include "skwyward-api-utils.hpp" +#include "spdlog/common.h" #include "spdlog/spdlog.h" #include "types.hpp" +#include +#include #include +#include #include #include #include #include #include +#include +#include +void batched_update(std::shared_ptr client_ptr, std::string base_uri){ + std::vector current_users = database_utils::get_all_users(client_ptr); + + + + const size_t batch_size = size_t {5}; + + for (size_t base_iteration = size_t{0}; current_users.size(); base_iteration+=batch_size) { + std::vector> futures_for_this_batch = std::vector>(); + + for (size_t j =base_iteration; j< base_iteration + batch_size && j past_record_unique = database_utils::load_latest_grades(client_ptr, current_record.user_id); + bool is_unique = database_utils::conditionally_insert_grades(client_ptr, current_record.user_id, grades); + if(!is_unique) return is_unique; + if(!past_record_unique.has_value()) return is_unique; + std::vector diffs = database_utils::diff_grade_responses(past_record_unique->response, grades); + std::optional current_record_unique = database_utils::load_latest_grades(client_ptr, current_record.user_id); + if(!current_record_unique.has_value()) return is_unique; + database_utils::insert_grade_updates(client_ptr, current_record.user_id, past_record_unique->response_id, current_record_unique->response_id, diffs); + spdlog::info("Fully inserted all diffs for user {}", current_record.user_id); + return is_unique; + })); + } + + for(std::future& future : futures_for_this_batch){ + spdlog::info("update logged: {}",future.get()); + } + } + + +} int main (int argc, char *argv[]) { + spdlog::set_level(spdlog::level::debug); auto config = toml::parse_file("config.toml"); std::string base_uri = config["host"].value_or(""); std::string test_username = config["test_username"].value_or(""); @@ -29,8 +73,8 @@ int main (int argc, char *argv[]) { api_utils::ErrorResponse test_login = api_methods::get_auth_status(base_uri, test_username, test_password); spdlog::info("Auth attempt response : {}", test_login.success); - api_utils::GradesResponse test_grades = api_methods::get_grades(base_uri, test_username, test_password); - spdlog::info("Grades: {}", nlohmann::json {test_grades}.dump()); + //api_utils::GradesResponse test_grades = api_methods::get_grades(base_uri, test_username, test_password); + //spdlog::info("Grades: {}", nlohmann::json {test_grades}.dump()); if(sodium_init()==-1){ spdlog::error("Sodium, the crypto lib, cannot be inited. bailing"); @@ -39,11 +83,25 @@ int main (int argc, char *argv[]) { std::shared_ptr client_shared_ptr = std::make_shared(clickhouse::ClientOptions().SetHost(clickhouse_host_name).SetPassword(clickhouse_password).SetUser(clickhouse_username).SetDefaultDatabase(clickhouse_schema)); - database_utils::register_user(client_shared_ptr, test_username, test_password); + // database_utils::register_user(client_shared_ptr, test_username, test_password); - spdlog::info("USER UUID: {}", database_utils::uuid_to_string( database_utils::get_user_uuid(client_shared_ptr, test_username).value() )); + // spdlog::info("USER UUID: {}", database_utils::uuid_to_string( database_utils::get_user_uuid(client_shared_ptr, test_username).value() )); spdlog::info("Connected to the database"); + + spdlog::info("runnign a thread"); + + std::thread update_thread = std::thread { [client_shared_ptr, base_uri](){ + while(true){ + batched_update(client_shared_ptr, base_uri); + sleep(5*60); + spdlog::info("Rerunning update cycle"); + } + } }; + + update_thread.join(); return 0; } + + diff --git a/src/skwyward-api-utils.cpp b/src/skwyward-api-utils.cpp index 63d7568..fa99c15 100644 --- a/src/skwyward-api-utils.cpp +++ b/src/skwyward-api-utils.cpp @@ -153,12 +153,21 @@ void to_json(json& j, const GradesResponse& r) { }; } -void from_json(const json& j, GradesResponse& r) { - j.at("success").get_to(r.success); - j.at("totalClasses").get_to(r.totalClasses); - j.at("grades").get_to(r.grades); +void from_json(const nlohmann::json& j, api_utils::GradesResponse& r) { + // success may or may not exist + r.success = j.value("success", false); + + r.totalClasses = j.value("totalClasses", 0); + + // grades MUST exist, but still guard + if (j.contains("grades") && j["grades"].is_array()) { + j.at("grades").get_to(r.grades); + } else { + r.grades.clear(); + } } + } // namespace api_utils @@ -191,15 +200,30 @@ api_utils::ErrorResponse get_auth_status( return nlohmann::json::parse(r.text) .get(); } - api_utils::GradesResponse get_grades(std::string url, std::string username, std::string password) { - api_utils::Login login = api_utils::Login{username, password}; +api_utils::GradesResponse get_grades( + std::string url, + std::string username, + std::string password +) { + api_utils::Login login{username, password}; + cpr::Response r = cpr::Post( - cpr::Url{url + "/fetch-grades"}, + cpr::Url{url + "/fetch-grades"}, cpr::Body{nlohmann::json(login).dump()}, cpr::Header{{"Content-Type", "application/json"}} ); - - return nlohmann::json::parse(r.text).get(); - } + + auto j = nlohmann::json::parse(r.text); + + if (j.is_array()) { + if (j.empty()) { + throw std::runtime_error("Grades API returned empty array"); + } + j = j.at(0); + } + + return j.get(); +} + } // namespace api_methods diff --git a/src/types.cpp b/src/types.cpp index 719224c..0b36b34 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -1,5 +1,8 @@ #include "types.hpp" #include "clickhouse/base/uuid.h" + +#include + #include #include #include @@ -10,19 +13,29 @@ using namespace clickhouse; namespace database_utils { // ---------------- UUID helpers ---------------- + clickhouse::UUID parse_uuid(const std::string& str) { - if (str.size() != 36) throw std::runtime_error("Invalid UUID string: " + str); + spdlog::debug("parse_uuid: input={}", str); + + if (str.size() != 36) + throw std::runtime_error("Invalid UUID string: " + str); std::string hexstr; - for (char c : str) if (c != '-') hexstr += c; + for (char c : str) + if (c != '-') hexstr += c; - if (hexstr.size() != 32) throw std::runtime_error("Invalid UUID format: " + str); + if (hexstr.size() != 32) + throw std::runtime_error("Invalid UUID format: " + str); uint64_t high = 0, low = 0; std::stringstream ss; + ss << std::hex << hexstr.substr(0, 16); ss >> high; - ss.clear(); ss.str(""); + + ss.clear(); + ss.str(""); + ss << std::hex << hexstr.substr(16, 16); ss >> low; @@ -33,17 +46,28 @@ std::string uuid_to_string(const clickhouse::UUID& u) { std::stringstream ss; ss << std::hex << std::setw(16) << std::setfill('0') << u.first << std::setw(16) << std::setfill('0') << u.second; + std::string s = ss.str(); - // Insert dashes for standard UUID format - return s.substr(0,8) + "-" + s.substr(8,4) + "-" + s.substr(12,4) + "-" + s.substr(16,4) + "-" + s.substr(20,12); + + return s.substr(0, 8) + "-" + + s.substr(8, 4) + "-" + + s.substr(12, 4) + "-" + + s.substr(16, 4) + "-" + + s.substr(20, 12); } // ---------------- Users ---------------- + std::vector get_all_users(const CHClient& client) { + spdlog::debug("get_all_users"); + std::vector out; + client->Select( "SELECT user_id, username, password FROM users", - [&](const Block& b){ + [&](const Block& b) { + spdlog::debug("get_all_users: rows={}", b.GetRowCount()); + for (size_t i = 0; i < b.GetRowCount(); ++i) { UserRecord u; u.user_id = uuid_to_string(b[0]->As()->At(i)); @@ -53,15 +77,22 @@ std::vector get_all_users(const CHClient& client) { } } ); + return out; } -bool register_user(const CHClient& client, const std::string& username, const std::string& password) { +bool register_user(const CHClient& client, + const std::string& username, + const std::string& password) { + spdlog::debug("register_user: username={}", username); + Block b; auto u = std::make_shared(); auto p = std::make_shared(); + u->Append(username); p->Append(password); + b.AppendColumn("username", u); b.AppendColumn("password", p); @@ -69,134 +100,199 @@ bool register_user(const CHClient& client, const std::string& username, const st return true; } -bool authenticate_user(const CHClient& client, const std::string& username, const std::string& password) { +bool authenticate_user(const CHClient& client, + const std::string& username, + const std::string& password) { + spdlog::debug("authenticate_user: username={}", username); + bool ok = false; + client->Select( - "SELECT count() FROM users WHERE username = '" + username + "' AND password = '" + password + "'", - [&](const Block& b){ ok = b[0]->As()->At(0) > 0; } + "SELECT count() FROM users " + "WHERE username = '" + username + + "' AND password = '" + password + "'", + [&](const Block& b) { + ok = b[0]->As()->At(0) > 0; + spdlog::debug("authenticate_user result={}", ok); + } ); + return ok; } // ---------------- Load latest snapshot ---------------- -std::optional load_latest_grades(const CHClient& client, const std::string& user_id) { + +std::optional +load_latest_grades(const CHClient& client, + const std::string& user_id) { + spdlog::debug("load_latest_grades: user_id={}", user_id); + GradeSnapshot snap; bool found = false; + client->Select( - "SELECT response_id, success, total_classes FROM grade_responses " - "WHERE user_id = '" + user_id + "' ORDER BY fetched_at DESC LIMIT 1", - [&](const Block& b){ - if (b.GetRowCount() == 0) return; + "SELECT response_id, success, total_classes " + "FROM grade_responses " + "WHERE user_id = '" + user_id + + "' ORDER BY fetched_at DESC LIMIT 1", + [&](const Block& b) { + if (b.GetRowCount() == 0) { + spdlog::debug("load_latest_grades: no previous snapshot"); + return; + } + found = true; - snap.response_id = uuid_to_string(b[0]->As()->At(0)); - snap.response.success = b[1]->As()->At(0); - snap.response.totalClasses = b[2]->As()->At(0); + snap.response_id = + uuid_to_string(b[0]->As()->At(0)); + snap.response.success = + b[1]->As()->At(0); + snap.response.totalClasses = + b[2]->As()->At(0); } ); - if (!found) return std::nullopt; + + if (!found) + return std::nullopt; + return snap; } // ---------------- Diff ---------------- -std::vector diff_grade_responses( - const api_utils::GradesResponse& old_resp, - const api_utils::GradesResponse& new_resp -) { - std::unordered_map old_map; + +std::vector +diff_grade_responses(const api_utils::GradesResponse& old_resp, + const api_utils::GradesResponse& new_resp) { + spdlog::debug("diff_grade_responses"); + + std::unordered_map old_map; + for (const auto& c : old_resp.grades) for (const auto& g : c.grades) old_map[c.className + "::" + g.name] = g; std::vector diffs; + for (const auto& c : new_resp.grades) { for (const auto& g : c.grades) { std::string key = c.className + "::" + g.name; + auto it = old_map.find(key); if (it == old_map.end()) { diffs.push_back({"", "", std::nullopt, g}); continue; } - if (it->second.score != g.score || it->second.attempts != g.attempts) { + + if (it->second.score != g.score || + it->second.attempts != g.attempts) { diffs.push_back({"", "", it->second, g}); } } } + + spdlog::debug("diff_grade_responses: diffs={}", diffs.size()); return diffs; } // ---------------- Conditional insert ---------------- -bool conditionally_insert_grades( - const CHClient& client, - const std::string& user_id, - const api_utils::GradesResponse& new_resp -) { + +bool conditionally_insert_grades(const CHClient& client, + const std::string& user_id, + const api_utils::GradesResponse& new_resp) { + spdlog::debug("conditionally_insert_grades: user_id={}", user_id); + auto old = load_latest_grades(client, user_id); - if (!old) return true; - return !diff_grade_responses(old->response, new_resp).empty(); + + if (!old) { + spdlog::debug( + "No previous snapshot — MUST insert first snapshot"); + return true; + } + + bool changed = + !diff_grade_responses(old->response, new_resp).empty(); + + spdlog::debug("conditionally_insert_grades: changed={}", changed); + return changed; } -void insert_grade_updates( - std::shared_ptr client, - const std::string& user_id_str, - const std::string& old_response_id_str, - const std::string& new_response_id_str, - const std::vector& diffs -) { - if (diffs.empty()) +// ---------------- Grade updates ---------------- + +void insert_grade_updates(const CHClient& client, + const std::string& user_id_str, + const std::string& old_response_id_str, + const std::string& new_response_id_str, + const std::vector& diffs) { + spdlog::debug("insert_grade_updates: diffs={}", diffs.size()); + + if (diffs.empty()) { + spdlog::debug("insert_grade_updates: nothing to insert"); return; + } Query query( "INSERT INTO grade_updates " - "(user_id, old_response_id, new_response_id, class_grade_id, assignment_id, assignment_name, " + "(user_id, old_response_id, new_response_id, " + "class_grade_id, assignment_id, assignment_name, " "old_score, new_score, old_attempts, new_attempts) " "VALUES " - "({user_id: String}, {old_response_id: String}, {new_response_id: String}, " - "{class_grade_id: String}, {assignment_id: String}, {assignment_name: String}, " - "{old_score: Nullable(String)}, {new_score: String}, " - "{old_attempts: Nullable(String)}, {new_attempts: String})" + "({user_id:String}, {old_response_id:String}, " + "{new_response_id:String}, {class_grade_id:String}, " + "{assignment_id:String}, {assignment_name:String}, " + "{old_score:Nullable(String)}, {new_score:String}, " + "{old_attempts:Nullable(String)}, {new_attempts:String})" ); - std::string user_uuid_str = uuid_to_string(parse_uuid(user_id_str)); - std::string old_response_uuid_str = uuid_to_string(parse_uuid(old_response_id_str)); - std::string new_response_uuid_str = uuid_to_string(parse_uuid(new_response_id_str)); - for (const auto& d : diffs) { - query.SetParam("user_id", user_uuid_str); - query.SetParam("old_response_id", old_response_uuid_str); - query.SetParam("new_response_id", new_response_uuid_str); + query.SetParam("user_id", user_id_str); + query.SetParam("old_response_id", old_response_id_str); + query.SetParam("new_response_id", new_response_id_str); - query.SetParam("class_grade_id", uuid_to_string(parse_uuid(d.class_grade_id))); - query.SetParam("assignment_id", uuid_to_string(parse_uuid(d.assignment_id))); + query.SetParam("class_grade_id", d.class_grade_id); + query.SetParam("assignment_id", d.assignment_id); query.SetParam("assignment_name", d.new_grade.name); if (d.old_grade) { - query.SetParam("old_score", std::to_string(d.old_grade->score)); - query.SetParam("old_attempts", d.old_grade->attempts); + query.SetParam("old_score", + std::to_string(d.old_grade->score)); + query.SetParam("old_attempts", + d.old_grade->attempts); } else { query.SetParam("old_score", QueryParamValue()); query.SetParam("old_attempts", QueryParamValue()); } - query.SetParam("new_score", std::to_string(d.new_grade.score)); - query.SetParam("new_attempts", d.new_grade.attempts); + query.SetParam("new_score", + std::to_string(d.new_grade.score)); + query.SetParam("new_attempts", + d.new_grade.attempts); client->Execute(query); } + + spdlog::debug("insert_grade_updates: done"); } +// ---------------- Lookup user UUID ---------------- + +std::optional +get_user_uuid(const CHClient& client, + const std::string& username) { + spdlog::debug("get_user_uuid: username={}", username); -std::optional get_user_uuid(const CHClient& client, const std::string& username) { std::optional result; client->Select( - "SELECT user_id FROM users WHERE username = '" + username + "' LIMIT 1", - [&](const clickhouse::Block& b) { - if (b.GetRowCount() == 0) + "SELECT user_id FROM users " + "WHERE username = '" + username + "' LIMIT 1", + [&](const Block& b) { + if (b.GetRowCount() == 0) { + spdlog::debug("get_user_uuid: not found"); return; + } - auto col_uuid = b[0]->As(); - auto uuid_val = col_uuid->At(0); - result = uuid_val; // your helper that converts UUID pair → string + result = b[0]->As()->At(0); + spdlog::debug("get_user_uuid: found"); } );