web: implement data migration, reorganize data folder

web
Alinson S. Xavier 3 months ago
parent cac9d7e230
commit 869498fa97

@ -7,7 +7,7 @@
import assert from "node:assert";
import { BusesColumnSpec, generateBusesData } from "./Buses";
import { generateCsv, parseCsv } from "../Common/Forms/DataTable";
import { TEST_DATA_1 } from "../../core/fixtures.test";
import { TEST_DATA_1 } from "../../core/Data/fixtures.test";
test("generate CSV", () => {
const [data, columns] = generateBusesData(TEST_DATA_1);

@ -14,7 +14,7 @@ import {
import { offerDownload } from "../Common/io";
import FileUploadElement from "../Common/Buttons/FileUploadElement";
import { useRef } from "react";
import { ValidationError } from "../../core/Validation/validate";
import { ValidationError } from "../../core/Data/validate";
import DataTable, {
ColumnSpec,
generateCsv,
@ -23,7 +23,6 @@ import DataTable, {
parseCsv,
} from "../Common/Forms/DataTable";
import { UnitCommitmentScenario } from "../../core/fixtures";
import { ColumnDefinition } from "tabulator-tables";
import {
changeBusData,
@ -32,6 +31,7 @@ import {
renameBus,
} from "../../core/Operations/busOps";
import { CaseBuilderSectionProps } from "./CaseBuilder";
import { UnitCommitmentScenario } from "../../core/Data/types";
export const BusesColumnSpec: ColumnSpec[] = [
{

@ -7,23 +7,20 @@
import Header from "./Header";
import Parameters from "./Parameters";
import BusesComponent from "./Buses";
import {
BLANK_SCENARIO,
TEST_SCENARIO,
UnitCommitmentScenario,
} from "../../core/fixtures";
import { BLANK_SCENARIO, TEST_SCENARIO } from "../../core/Data/fixtures";
import "tabulator-tables/dist/css/tabulator.min.css";
import "../Common/Forms/Tables.css";
import { useState } from "react";
import Footer from "./Footer";
import { validate } from "../../core/Validation/validate";
import { validate } from "../../core/Data/validate";
import { offerDownload } from "../Common/io";
import { preprocess } from "../../core/Operations/preprocessing";
import Toast from "../Common/Forms/Toast";
import ProfiledUnitsComponent from "./ProfiledUnits";
import ThermalUnitsComponent from "./ThermalUnits";
import TransmissionLinesComponent from "./TransmissionLines";
import { UnitCommitmentScenario } from "../../core/Data/types";
export interface CaseBuilderSectionProps {
scenario: UnitCommitmentScenario;

@ -6,9 +6,9 @@
import styles from "./Header.module.css";
import SiteHeaderButton from "../Common/Buttons/SiteHeaderButton";
import { UnitCommitmentScenario } from "../../core/fixtures";
import { useRef } from "react";
import FileUploadElement from "../Common/Buttons/FileUploadElement";
import { UnitCommitmentScenario } from "../../core/Data/types";
interface HeaderProps {
onClear: () => void;

@ -7,12 +7,12 @@
import SectionHeader from "../Common/SectionHeader/SectionHeader";
import Form from "../Common/Forms/Form";
import TextInputRow from "../Common/Forms/TextInputRow";
import { UnitCommitmentScenario } from "../../core/fixtures";
import {
changeParameter,
changeTimeHorizon,
changeTimeStep,
} from "../../core/Operations/parameterOps";
import { UnitCommitmentScenario } from "../../core/Data/types";
interface ParametersProps {
scenario: UnitCommitmentScenario;

@ -13,12 +13,12 @@ import {
parseProfiledUnitsCsv,
ProfiledUnitsColumnSpec,
} from "./ProfiledUnits";
import { TEST_DATA_1 } from "../../core/fixtures.test";
import { TEST_DATA_1 } from "../../core/Data/fixtures.test";
import assert from "node:assert";
import {
getProfiledGenerators,
getThermalGenerators,
} from "../../core/fixtures";
} from "../../core/Data/types";
test("parse CSV", () => {
const csvContents =

@ -18,11 +18,6 @@ import DataTable, {
generateTableData,
parseCsv,
} from "../Common/Forms/DataTable";
import {
getProfiledGenerators,
getThermalGenerators,
UnitCommitmentScenario,
} from "../../core/fixtures";
import { ColumnDefinition } from "tabulator-tables";
import { offerDownload } from "../Common/io";
import FileUploadElement from "../Common/Buttons/FileUploadElement";
@ -33,8 +28,13 @@ import {
deleteGenerator,
renameGenerator,
} from "../../core/Operations/generatorOps";
import { ValidationError } from "../../core/Validation/validate";
import { ValidationError } from "../../core/Data/validate";
import { CaseBuilderSectionProps } from "./CaseBuilder";
import {
getProfiledGenerators,
getThermalGenerators,
UnitCommitmentScenario,
} from "../../core/Data/types";
export const ProfiledUnitsColumnSpec: ColumnSpec[] = [
{

@ -10,7 +10,7 @@ import {
generateTableColumns,
generateTableData,
} from "../Common/Forms/DataTable";
import { TEST_DATA_1 } from "../../core/fixtures.test";
import { TEST_DATA_1 } from "../../core/Data/fixtures.test";
import {
generateThermalUnitsData,
parseThermalUnitsCsv,
@ -20,7 +20,7 @@ import assert from "node:assert";
import {
getProfiledGenerators,
getThermalGenerators,
} from "../../core/fixtures";
} from "../../core/Data/types";
test("generateTableColumns", () => {
const columns = generateTableColumns(TEST_DATA_1, ThermalUnitsColumnSpec);

@ -14,7 +14,7 @@ import DataTable, {
import { CaseBuilderSectionProps } from "./CaseBuilder";
import { useRef } from "react";
import FileUploadElement from "../Common/Buttons/FileUploadElement";
import { ValidationError } from "../../core/Validation/validate";
import { ValidationError } from "../../core/Data/validate";
import SectionHeader from "../Common/SectionHeader/SectionHeader";
import SectionButton from "../Common/Buttons/SectionButton";
import {
@ -22,11 +22,6 @@ import {
faPlus,
faUpload,
} from "@fortawesome/free-solid-svg-icons";
import {
getProfiledGenerators,
getThermalGenerators,
UnitCommitmentScenario,
} from "../../core/fixtures";
import { ColumnDefinition } from "tabulator-tables";
import { offerDownload } from "../Common/io";
import {
@ -35,6 +30,11 @@ import {
deleteGenerator,
renameGenerator,
} from "../../core/Operations/generatorOps";
import {
getProfiledGenerators,
getThermalGenerators,
UnitCommitmentScenario,
} from "../../core/Data/types";
export const ThermalUnitsColumnSpec: ColumnSpec[] = [
{

@ -18,11 +18,10 @@ import DataTable, {
generateTableData,
parseCsv,
} from "../Common/Forms/DataTable";
import { UnitCommitmentScenario } from "../../core/fixtures";
import { ColumnDefinition } from "tabulator-tables";
import FileUploadElement from "../Common/Buttons/FileUploadElement";
import { useRef } from "react";
import { ValidationError } from "../../core/Validation/validate";
import { ValidationError } from "../../core/Data/validate";
import { CaseBuilderSectionProps } from "./CaseBuilder";
import {
changeTransmissionLineData,
@ -31,6 +30,7 @@ import {
renameTransmissionLine,
} from "../../core/Operations/transmissionOps";
import { offerDownload } from "../Common/io";
import { UnitCommitmentScenario } from "../../core/Data/types";
export const TransmissionLinesColumnSpec: ColumnSpec[] = [
{

@ -10,10 +10,10 @@ import {
ColumnDefinition,
TabulatorFull as Tabulator,
} from "tabulator-tables";
import { ValidationError } from "../../../core/Validation/validate";
import { UnitCommitmentScenario } from "../../../core/fixtures";
import { ValidationError } from "../../../core/Data/validate";
import Papa from "papaparse";
import { parseBool, parseNumber } from "../../../core/Operations/commonOps";
import { UnitCommitmentScenario } from "../../../core/Data/types";
export interface ColumnSpec {
title: string;

@ -7,7 +7,7 @@
import formStyles from "./Form.module.css";
import HelpButton from "../Buttons/HelpButton";
import React, { useRef, useState } from "react";
import { ValidationError } from "../../../core/Validation/validate";
import { ValidationError } from "../../../core/Data/validate";
interface TextInputRowProps {
label: string;

@ -1,4 +1,10 @@
import { UnitCommitmentScenario } from "./fixtures";
/*
* UnitCommitment.jl: Optimization Package for Security-Constrained Unit Commitment
* Copyright (C) 2020-2025, UChicago Argonne, LLC. All rights reserved.
* Released under the modified BSD license. See COPYING.md for more details.
*/
import { UnitCommitmentScenario } from "./types";
export const TEST_DATA_1: UnitCommitmentScenario = {
Parameters: {

@ -4,86 +4,12 @@
* Released under the modified BSD license. See COPYING.md for more details.
*/
import { UnitCommitmentScenario } from "./types";
export interface Buses {
[busName: string]: { "Load (MW)": number[] };
}
export interface Generators {
[name: string]: ProfiledUnit | ThermalUnit;
}
export interface ProfiledUnit {
Bus: string;
Type: "Profiled";
"Minimum power (MW)": number[];
"Maximum power (MW)": number[];
"Cost ($/MW)": number;
}
export interface ThermalUnit {
Bus: string;
Type: "Thermal";
"Production cost curve (MW)": number[];
"Production cost curve ($)": number[];
"Startup costs ($)": number[];
"Startup delays (h)": number[];
"Ramp up limit (MW)": number | "";
"Ramp down limit (MW)": number | "";
"Startup limit (MW)": number | "";
"Shutdown limit (MW)": number | "";
"Minimum downtime (h)": number;
"Minimum uptime (h)": number;
"Initial status (h)": number;
"Initial power (MW)": number;
"Must run?": boolean;
}
export interface TransmissionLine {
"Source bus": string;
"Target bus": string;
"Susceptance (S)": number;
"Normal flow limit (MW)": number;
"Emergency flow limit (MW)": number;
"Flow limit penalty ($/MW)": number;
}
export interface UnitCommitmentScenario {
Parameters: {
Version: string;
"Power balance penalty ($/MW)": number;
"Time horizon (h)": number;
"Time step (min)": number;
};
Buses: Buses;
Generators: Generators;
"Transmission lines": {
[name: string]: TransmissionLine;
};
}
const getTypedGenerators = <T extends any>(
scenario: UnitCommitmentScenario,
type: string,
): {
[key: string]: T;
} => {
const selected: { [key: string]: T } = {};
for (const [name, gen] of Object.entries(scenario.Generators)) {
if (gen["Type"] === type) selected[name] = gen as T;
}
return selected;
};
export const getProfiledGenerators = (
scenario: UnitCommitmentScenario,
): { [key: string]: ProfiledUnit } =>
getTypedGenerators<ProfiledUnit>(scenario, "Profiled");
export const getThermalGenerators = (
scenario: UnitCommitmentScenario,
): { [key: string]: ThermalUnit } =>
getTypedGenerators<ThermalUnit>(scenario, "Thermal");
export const BLANK_SCENARIO: UnitCommitmentScenario = {
Parameters: {
Version: "0.4",

@ -0,0 +1,34 @@
/*
* UnitCommitment.jl: Optimization Package for Security-Constrained Unit Commitment
* Copyright (C) 2020-2025, UChicago Argonne, LLC. All rights reserved.
* Released under the modified BSD license. See COPYING.md for more details.
*/
import assert from "node:assert";
import fs from "node:fs";
import pako from "pako";
import { migrateToV03, migrateToV04 } from "./migrate";
function readJsonGz(filename: string) {
const compressedData = fs.readFileSync(filename);
const decompressedData = pako.inflate(compressedData, { to: "string" });
return JSON.parse(decompressedData);
}
test("migrateToV03", () => {
const jsonData = readJsonGz("../test/fixtures/ucjl-0.2.json.gz");
migrateToV03(jsonData);
assert.deepEqual(jsonData.Reserves, {
r1: {
"Amount (MW)": 100,
"Shortfall penalty ($/MW)": 1000,
Type: "spinning",
},
});
});
test("migrateToV04", () => {
const jsonData = readJsonGz("../test/fixtures/ucjl-0.3.json.gz");
migrateToV04(jsonData);
assert.equal(jsonData.Generators["g1"].Type, "Thermal");
});

@ -0,0 +1,56 @@
/*
* UnitCommitment.jl: Optimization Package for Security-Constrained Unit Commitment
* Copyright (C) 2020-2025, UChicago Argonne, LLC. All rights reserved.
* Released under the modified BSD license. See COPYING.md for more details.
*/
import { ValidationError } from "./validate";
export const migrate = (json: any): ValidationError | null => {
const version = json.Parameters?.Version;
if (!version) {
return {
message:
"The provided input file cannot be loaded because it does not " +
"specify what version of UnitCommitment.jl it was written for.",
};
}
if (!["0.2", "0.3", "0.4"].includes(version)) {
return { message: `Unsupported file version: ${version}` };
}
if (version < "0.3") migrateToV03(json);
if (version < "0.4") migrateToV04(json);
json.Parameters.Version = "0.4";
return null;
};
export const migrateToV03 = (json: any): void => {
if (json.Reserves && json.Reserves["Spinning (MW)"] != null) {
const amount = json.Reserves["Spinning (MW)"];
json.Reserves = {
r1: {
Type: "spinning",
"Amount (MW)": amount,
},
};
if (json.Generators) {
for (const genName in json.Generators) {
const gen = json.Generators[genName];
if (gen["Provides spinning reserves?"] === true) {
gen["Reserve eligibility"] = ["r1"];
}
}
}
}
};
export const migrateToV04 = (json: any): void => {
if (json.Generators) {
for (const genName in json.Generators) {
const gen = json.Generators[genName];
if (gen.Type == null) {
gen.Type = "Thermal";
}
}
}
};

@ -0,0 +1,81 @@
/*
* UnitCommitment.jl: Optimization Package for Security-Constrained Unit Commitment
* Copyright (C) 2020-2025, UChicago Argonne, LLC. All rights reserved.
* Released under the modified BSD license. See COPYING.md for more details.
*/
import { Buses } from "./fixtures";
export interface Generators {
[name: string]: ProfiledUnit | ThermalUnit;
}
export interface ProfiledUnit {
Bus: string;
Type: "Profiled";
"Minimum power (MW)": number[];
"Maximum power (MW)": number[];
"Cost ($/MW)": number;
}
export interface ThermalUnit {
Bus: string;
Type: "Thermal";
"Production cost curve (MW)": number[];
"Production cost curve ($)": number[];
"Startup costs ($)": number[];
"Startup delays (h)": number[];
"Ramp up limit (MW)": number | "";
"Ramp down limit (MW)": number | "";
"Startup limit (MW)": number | "";
"Shutdown limit (MW)": number | "";
"Minimum downtime (h)": number;
"Minimum uptime (h)": number;
"Initial status (h)": number;
"Initial power (MW)": number;
"Must run?": boolean;
}
export interface TransmissionLine {
"Source bus": string;
"Target bus": string;
"Susceptance (S)": number;
"Normal flow limit (MW)": number;
"Emergency flow limit (MW)": number;
"Flow limit penalty ($/MW)": number;
}
export interface UnitCommitmentScenario {
Parameters: {
Version: string;
"Power balance penalty ($/MW)": number;
"Time horizon (h)": number;
"Time step (min)": number;
};
Buses: Buses;
Generators: Generators;
"Transmission lines": {
[name: string]: TransmissionLine;
};
}
const getTypedGenerators = <T extends any>(
scenario: UnitCommitmentScenario,
type: string,
): {
[key: string]: T;
} => {
const selected: { [key: string]: T } = {};
for (const [name, gen] of Object.entries(scenario.Generators)) {
if (gen["Type"] === type) selected[name] = gen as T;
}
return selected;
};
export const getProfiledGenerators = (
scenario: UnitCommitmentScenario,
): { [key: string]: ProfiledUnit } =>
getTypedGenerators<ProfiledUnit>(scenario, "Profiled");
export const getThermalGenerators = (
scenario: UnitCommitmentScenario,
): { [key: string]: ThermalUnit } =>
getTypedGenerators<ThermalUnit>(scenario, "Thermal");

@ -6,7 +6,7 @@
import { changeBusData, createBus, deleteBus, renameBus } from "./busOps";
import assert from "node:assert";
import { TEST_DATA_1 } from "../fixtures.test";
import { TEST_DATA_1 } from "../Data/fixtures.test";
test("createBus", () => {
const newScenario = createBus(TEST_DATA_1);

@ -4,8 +4,8 @@
* Released under the modified BSD license. See COPYING.md for more details.
*/
import { Buses, UnitCommitmentScenario } from "../fixtures";
import { ValidationError } from "../Validation/validate";
import { Buses } from "../Data/fixtures";
import { ValidationError } from "../Data/validate";
import { generateTimeslots } from "../../components/Common/Forms/DataTable";
import {
changeData,
@ -13,6 +13,7 @@ import {
renameItemInObject,
} from "./commonOps";
import { BusesColumnSpec } from "../../components/CaseBuilder/Buses";
import { UnitCommitmentScenario } from "../Data/types";
export const createBus = (scenario: UnitCommitmentScenario) => {
const name = generateUniqueName(scenario.Buses, "b");

@ -4,9 +4,9 @@
* Released under the modified BSD license. See COPYING.md for more details.
*/
import { ValidationError } from "../Validation/validate";
import { UnitCommitmentScenario } from "../fixtures";
import { ValidationError } from "../Data/validate";
import { ColumnSpec } from "../../components/Common/Forms/DataTable";
import { UnitCommitmentScenario } from "../Data/types";
export const renameItemInObject = <T>(
oldName: string,

@ -4,7 +4,7 @@
* Released under the modified BSD license. See COPYING.md for more details.
*/
import { TEST_DATA_1, TEST_DATA_BLANK } from "../fixtures.test";
import { TEST_DATA_1, TEST_DATA_BLANK } from "../Data/fixtures.test";
import assert from "node:assert";
import {
changeProfiledUnitData,
@ -14,7 +14,7 @@ import {
deleteGenerator,
renameGenerator,
} from "./generatorOps";
import { ValidationError } from "../Validation/validate";
import { ValidationError } from "../Data/validate";
test("createProfiledUnit", () => {
const [newScenario, err] = createProfiledUnit(TEST_DATA_1);

@ -4,9 +4,8 @@
* Released under the modified BSD license. See COPYING.md for more details.
*/
import { Generators, UnitCommitmentScenario } from "../fixtures";
import { generateTimeslots } from "../../components/Common/Forms/DataTable";
import { ValidationError } from "../Validation/validate";
import { ValidationError } from "../Data/validate";
import {
assertBusesNotEmpty,
changeData,
@ -15,6 +14,7 @@ import {
} from "./commonOps";
import { ProfiledUnitsColumnSpec } from "../../components/CaseBuilder/ProfiledUnits";
import { ThermalUnitsColumnSpec } from "../../components/CaseBuilder/ThermalUnits";
import { Generators, UnitCommitmentScenario } from "../Data/types";
export const createProfiledUnit = (
scenario: UnitCommitmentScenario,

@ -10,7 +10,7 @@ import {
evaluatePwlFunction,
} from "./parameterOps";
import assert from "node:assert";
import { TEST_DATA_1, TEST_DATA_2 } from "../fixtures.test";
import { TEST_DATA_1, TEST_DATA_2 } from "../Data/fixtures.test";
test("changeTimeHorizon: Shrink 1", () => {
const [newScenario, err] = changeTimeHorizon(TEST_DATA_1, "3");

@ -4,8 +4,9 @@
* Released under the modified BSD license. See COPYING.md for more details.
*/
import { Buses, UnitCommitmentScenario } from "../fixtures";
import { ValidationError } from "../Validation/validate";
import { Buses } from "../Data/fixtures";
import { ValidationError } from "../Data/validate";
import { UnitCommitmentScenario } from "../Data/types";
export const changeTimeHorizon = (
scenario: UnitCommitmentScenario,

@ -6,7 +6,7 @@
* Released under the modified BSD license. See COPYING.md for more details.
*/
import { validate } from "../Validation/validate";
import { validate } from "../Data/validate";
export const preprocess = (data) => {
// Make a copy of the original data

@ -4,7 +4,7 @@
* Released under the modified BSD license. See COPYING.md for more details.
*/
import { TEST_DATA_1 } from "../fixtures.test";
import { TEST_DATA_1 } from "../Data/fixtures.test";
import assert from "node:assert";
import {
changeTransmissionLineData,
@ -12,7 +12,7 @@ import {
deleteTransmissionLine,
renameTransmissionLine,
} from "./transmissionOps";
import { ValidationError } from "../Validation/validate";
import { ValidationError } from "../Data/validate";
test("createTransmissionLine", () => {
const [newScenario, err] = createTransmissionLine(TEST_DATA_1);

@ -4,15 +4,15 @@
* Released under the modified BSD license. See COPYING.md for more details.
*/
import { TransmissionLine, UnitCommitmentScenario } from "../fixtures";
import {
assertBusesNotEmpty,
changeData,
generateUniqueName,
renameItemInObject,
} from "./commonOps";
import { ValidationError } from "../Validation/validate";
import { ValidationError } from "../Data/validate";
import { TransmissionLinesColumnSpec } from "../../components/CaseBuilder/TransmissionLines";
import { TransmissionLine, UnitCommitmentScenario } from "../Data/types";
export const createTransmissionLine = (
scenario: UnitCommitmentScenario,

Loading…
Cancel
Save