luis/coins
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Add sqlite database from json files

Browse Source
This commit is contained in:
Luís Alves
2023-09-22 10:45:43 +01:00
parent b205bb1444
commit 93ea54f300
6 changed files with 1414 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/init.sql
+171
View File
@@ -0,0 +1,171 @@
CREATE TABLE Issuer (
code TEXT NOT NULL COLLATE NOCASE PRIMARY KEY,
name TEXT NOT NULL COLLATE NOCASE
);
CREATE TABLE Currency (
id INTEGER NOT NULL PRIMARY KEY,
name TEXT NOT NULL COLLATE NOCASE,
full_name TEXT NOT NULL COLLATE NOCASE
);
CREATE TABLE Value (
id INTEGER NOT NULL PRIMARY KEY,
text TEXT COLLATE NOCASE,
numeric_value REAL,
numerator INTEGER,
denominator INTEGER,
currency_id INTEGER,
FOREIGN KEY (currency_id) REFERENCES Currency(id)
);
CREATE TABLE Demonetization (
id INTEGER NOT NULL PRIMARY KEY,
is_demonetized BOOLEAN NOT NULL,
demonetization_date TEXT
);
CREATE TABLE Composition (
id INTEGER NOT NULL PRIMARY KEY,
text TEXT
);
CREATE TABLE Technique (
id INTEGER NOT NULL PRIMARY KEY,
text TEXT
);
CREATE TABLE Reverse (
id INTEGER NOT NULL PRIMARY KEY,
engravers TEXT,
designers TEXT,
description TEXT,
lettering TEXT,
lettering_scripts TEXT,
unabridged_legend TEXT,
lettering_translation TEXT,
picture TEXT,
thumbnail TEXT,
picture_copyright TEXT,
picture_copyright_url TEXT,
picture_license_name TEXT,
picture_license_url TEXT
);
CREATE TABLE Obverse (
id INTEGER NOT NULL PRIMARY KEY,
engravers TEXT,
designers TEXT,
description TEXT,
lettering TEXT,
lettering_scripts TEXT,
unabridged_legend TEXT,
lettering_translation TEXT,
picture TEXT,
thumbnail TEXT,
picture_copyright TEXT,
picture_copyright_url TEXT,
picture_license_name TEXT,
picture_license_url TEXT
);
CREATE TABLE Edge (
id INTEGER NOT NULL PRIMARY KEY,
engravers TEXT,
designers TEXT,
description TEXT,
lettering TEXT,
lettering_scripts TEXT,
unabridged_legend TEXT,
lettering_translation TEXT,
picture TEXT,
thumbnail TEXT,
picture_copyright TEXT,
picture_copyright_url TEXT,
picture_license_name TEXT,
picture_license_url TEXT
);
CREATE TABLE Watermark (
id INTEGER NOT NULL PRIMARY KEY,
engravers TEXT,
designers TEXT,
description TEXT,
lettering TEXT,
lettering_scripts TEXT,
unabridged_legend TEXT,
lettering_translation TEXT,
picture TEXT,
thumbnail TEXT,
picture_copyright TEXT,
picture_copyright_url TEXT,
picture_license_name TEXT,
picture_license_url TEXT
);
CREATE TABLE NumistaType (
id INTEGER NOT NULL PRIMARY KEY,
url TEXT,
title TEXT NOT NULL COLLATE NOCASE,
category TEXT NOT NULL CHECK (category IN ('coin', 'banknote', 'exonumia')),
issuer_code TEXT,
min_year INTEGER,
max_year INTEGER,
type TEXT COLLATE NOCASE,
value_id INTEGER,
demonetization_id INTEGER,
shape TEXT,
composition_id INTEGER,
technique_id INTEGER,
weight REAL,
size REAL,
thickness REAL,
orientation TEXT CHECK (orientation IN ('coin', 'medal', 'variable', 'three', 'nine')),
obverse_id INTEGER,
reverse_id INTEGER,
edge_id INTEGER,
watermark_id INTEGER,
series TEXT,
commemorated_topic TEXT,
comments TEXT,
count INTEGER NOT NULL,
color TEXT,
FOREIGN KEY (issuer_code) REFERENCES Issuer(code),
FOREIGN KEY (value_id) REFERENCES Value(id),
FOREIGN KEY (demonetization_id) REFERENCES Demonetization(id),
FOREIGN KEY (composition_id) REFERENCES Composition(id),
FOREIGN KEY (technique_id) REFERENCES Technique(id),
FOREIGN KEY (obverse_id) REFERENCES Obverse(id),
FOREIGN KEY (reverse_id) REFERENCES Reverse(id),
FOREIGN KEY (edge_id) REFERENCES Edge(id),
FOREIGN KEY (watermark_id) REFERENCES Watermark(id)
);
CREATE TABLE RulerGroup (
id INTEGER NOT NULL PRIMARY KEY,
name TEXT NOT NULL
);
CREATE TABLE Ruler (
id INTEGER NOT NULL PRIMARY KEY,
name TEXT NOT NULL,
wikidata_id TEXT,
group_id INTEGER,
FOREIGN KEY (group_id) REFERENCES RulerGroup(id)
);
CREATE TABLE TypeRuler (
type_id INTEGER NOT NULL,
ruler_id INTEGER NOT NULL,
FOREIGN KEY (type_id) REFERENCES NumistaType(id),
FOREIGN KEY (ruler_id) REFERENCES Ruler(id),
UNIQUE(type_id, ruler_id)
);
CREATE TABLE TypeRelated (
type1_id INTEGER NOT NULL,
type2_id INTEGER NOT NULL,
FOREIGN KEY (type1_id) REFERENCES NumistaType(id),
FOREIGN KEY (type2_id) REFERENCES NumistaType(id),
UNIQUE(type1_id, type2_id)
);
bin/insert.mjs
+204
View File
@@ -0,0 +1,204 @@
import fs from "fs/promises";
import path from "path";
import Database from "better-sqlite3";
const db = new Database("main.db");
db.pragma("journal_mode = WAL");
const omit = (data, keys = []) =>
Object.fromEntries(
Object.entries(data).filter(
([key, value]) =>
!keys.includes(key) &&
(typeof value === "number" || typeof value === "boolean" || !!value),
),
);
const exists = (table, primaryKey, value) => {
const stmt = `SELECT COUNT(*) count FROM ${table} WHERE ${primaryKey}=${
typeof value === "string" ? `'${value}'` : value
}`;
return db.prepare(stmt).get().count > 0;
};
const inserter = (table, data, primaryKey) => {
console.log();
if (primaryKey && exists(table, primaryKey, data[primaryKey])) {
console.log(`${table} ${data[primaryKey]} already exists`);
return;
}
const entries = Object.entries(data);
const stmt = `INSERT INTO ${table} (${entries
.map(([key]) => key)
.join(", ")}) VALUES (${entries
.map(([, value]) => {
if (Array.isArray(value)) {
return `'${value.join(", ").replaceAll("'", "''")}'`;
}
if (typeof value === "string") {
return `'${value.replaceAll("'", "''")}'`;
}
return value;
})
.join(", ")});`;
console.log(stmt);
db.exec(stmt);
};
const handleType = (type) => {
if (type.issuer) {
inserter("Issuer", omit(type.issuer), "code");
}
if (type.value) {
if (type.value.currency) {
inserter("Currency", omit(type.value.currency), "id");
}
inserter(
"Value",
omit(
{
...type.value,
currency_id: type.value.currency?.id,
id: type.id,
},
["currency"],
),
"id",
);
}
if (type.demonetization) {
inserter(
"Demonetization",
omit({ ...type.demonetization, id: type.id }),
"id",
);
}
if (type.composition) {
inserter("Composition", omit({ ...type.composition, id: type.id }), "id");
}
if (type.technique) {
inserter("Technique", omit({ ...type.technique, id: type.id }), "id");
}
if (type.obverse) {
inserter("Obverse", omit({ ...type.obverse, id: type.id }), "id");
}
if (type.reverse) {
inserter("Reverse", omit({ ...type.reverse, id: type.id }), "id");
}
if (type.edge) {
inserter("Edge", omit({ ...type.edge, id: type.id }), "id");
}
if (type.watermark) {
inserter("Watermark", omit({ ...type.watermark, id: type.id }), "id");
}
inserter(
"NumistaType",
omit(
{
...type,
issuer_code: type.issuer?.code,
count: type.count || 1,
...(type.value && { value_id: type.id }),
...(type.demonetization && { demonetization_id: type.id }),
...(type.composition && { composition_id: type.id }),
...(type.technique && { technique_id: type.id }),
...(type.obverse && { obverse_id: type.id }),
...(type.reverse && { reverse_id: type.id }),
...(type.edge && { edge_id: type.id }),
...(type.watermark && { watermark_id: type.id }),
},
[
"issuer",
"value",
"ruler",
"obverse",
"reverse",
"edge",
"watermark",
"tags",
"references",
"mints",
"printers",
"demonetization",
"composition",
"technique",
"related_types",
],
),
"id",
);
if (type.ruler?.length > 0) {
for (const ruler of type.ruler) {
if (ruler.group) {
inserter("RulerGroup", omit(ruler.group), "id");
}
inserter(
"Ruler",
omit(
{
...ruler,
group_id: ruler.group?.id,
},
["group"],
),
"id",
);
try {
inserter("TypeRuler", {
type_id: type.id,
ruler_id: ruler.id,
});
} catch {
//ok
}
}
}
};
const handleRelated = (type) => {
if (type.related_types?.length > 0) {
for (const related_type of type.related_types) {
if (exists("NumistaType", "id", related_type.id)) {
inserter(
"TypeRelated",
omit({
type1_id: type.id,
type2_id: related_type.id,
}),
);
}
}
}
};
const forEachType = async (callback) => {
for (const typeRoot of ["coins", "banknotes"]) {
for (const country of await fs.readdir(path.join("../public", typeRoot))) {
for (const typePath of (
await fs.readdir(path.join("../public", typeRoot, country))
).filter((filepath) => filepath.endsWith(".json"))) {
const type = JSON.parse(
await fs.readFile(
path.join("../public", typeRoot, country, typePath),
"utf-8",
),
);
callback(type);
}
}
}
};
await forEachType(handleType);
await forEachType(handleRelated);
db.close();
package.json
+4 -1
View File
@@ -10,11 +10,14 @@
"get-type": "./bin/get-type.sh" "get-type": "./bin/get-type.sh"
}, },
"dependencies": { "dependencies": {
"@prisma/client": "5.3.1",
"@react-three/cannon": "^6.6.0", "@react-three/cannon": "^6.6.0",
"@react-three/drei": "^9.83.9", "@react-three/drei": "^9.83.9",
"@react-three/fiber": "^8.13.0", "@react-three/fiber": "^8.13.0",
"next": "13.4.19", "better-sqlite3": "^8.6.0",
"next": "^13.5.1",
"prettier": "^3.0.3", "prettier": "^3.0.3",
"prisma": "^5.3.1",
"react": "18.2.0", "react": "18.2.0",
"react-div-100vh": "^0.7.0", "react-div-100vh": "^0.7.0",
"react-dom": "18.2.0", "react-dom": "18.2.0",
prisma/main.db
BIN
View File
Binary file not shown.
prisma/schema.prisma
+197
View File
@@ -0,0 +1,197 @@
generator client {
provider = "prisma-client-js"
}
datasource db {
provider = "sqlite"
url = "file:./main.db"
}
model Composition {
id Int @id @default(autoincrement())
text String?
type NumistaType[]
}
model Currency {
id Int @id @default(autoincrement())
name String
full_name String
value Value[]
}
model Demonetization {
id Int @id @default(autoincrement())
is_demonetized Boolean
demonetization_date String?
type NumistaType[]
}
model Edge {
id Int @id @default(autoincrement())
engravers String?
designers String?
description String?
lettering String?
lettering_scripts String?
unabridged_legend String?
lettering_translation String?
picture String?
thumbnail String?
picture_copyright String?
picture_copyright_url String?
picture_license_name String?
picture_license_url String?
type NumistaType[]
}
model Issuer {
code String @id
name String
type NumistaType[]
}
model NumistaType {
id Int @id @default(autoincrement())
url String?
title String
category String
issuer_code String?
min_year Int?
max_year Int?
type String?
value_id Int?
demonetization_id Int?
shape String?
composition_id Int?
technique_id Int?
weight Float?
size Float?
thickness Float?
orientation String?
obverse_id Int?
reverse_id Int?
edge_id Int?
watermark_id Int?
series String?
commemorated_topic String?
comments String?
count Int @default(1)
color String?
watermark Watermark? @relation(fields: [watermark_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
edge Edge? @relation(fields: [edge_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
reverse Reverse? @relation(fields: [reverse_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
obverse Obverse? @relation(fields: [obverse_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
technique Technique? @relation(fields: [technique_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
composition Composition? @relation(fields: [composition_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
demonetization Demonetization? @relation(fields: [demonetization_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
value Value? @relation(fields: [value_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
issuer Issuer? @relation(fields: [issuer_code], references: [code], onDelete: NoAction, onUpdate: NoAction)
relatedB TypeRelated[] @relation("TypeRelated_type2_idToType")
relatedA TypeRelated[] @relation("TypeRelated_type1_idToType")
rulers TypeRuler[]
}
model Obverse {
id Int @id @default(autoincrement())
engravers String?
designers String?
description String?
lettering String?
lettering_scripts String?
unabridged_legend String?
lettering_translation String?
picture String?
thumbnail String?
picture_copyright String?
picture_copyright_url String?
picture_license_name String?
picture_license_url String?
type NumistaType[]
}
model Reverse {
id Int @id @default(autoincrement())
engravers String?
designers String?
description String?
lettering String?
lettering_scripts String?
unabridged_legend String?
lettering_translation String?
picture String?
thumbnail String?
picture_copyright String?
picture_copyright_url String?
picture_license_name String?
picture_license_url String?
type NumistaType[]
}
model Ruler {
id Int @id @default(autoincrement())
name String
wikidata_id String?
group_id Int?
group RulerGroup? @relation(fields: [group_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
TypeRuler TypeRuler[]
}
model RulerGroup {
id Int @id @default(autoincrement())
name String
rulers Ruler[]
}
model Technique {
id Int @id @default(autoincrement())
text String?
type NumistaType[]
}
model TypeRelated {
type1_id Int
type2_id Int
type1 NumistaType @relation("TypeRelated_type2_idToType", fields: [type2_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
type2 NumistaType @relation("TypeRelated_type1_idToType", fields: [type1_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
@@unique([type1_id, type2_id], map: "sqlite_autoindex_TypeRelated_1")
}
model TypeRuler {
type_id Int
ruler_id Int
ruler Ruler @relation(fields: [ruler_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
type NumistaType @relation(fields: [type_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
@@unique([type_id, ruler_id], map: "sqlite_autoindex_TypeRuler_1")
}
model Value {
id Int @id @default(autoincrement())
text String?
numeric_value Float?
numerator Int?
denominator Int?
currency_id Int?
type NumistaType[]
currency Currency? @relation(fields: [currency_id], references: [id], onDelete: NoAction, onUpdate: NoAction)
}
model Watermark {
id Int @id @default(autoincrement())
engravers String?
designers String?
description String?
lettering String?
lettering_scripts String?
unabridged_legend String?
lettering_translation String?
picture String?
thumbnail String?
picture_copyright String?
picture_copyright_url String?
picture_license_name String?
picture_license_url String?
type NumistaType[]
}
yarn.lock
+838 -76
View File
File diff suppressed because it is too large Load Diff