[feat] read tcgcollector csv

package-lock.json

@@ -16,6 +16,7 @@
         "bootstrap": "^5.3.8",
         "chalk": "^5.6.2",
         "chart.js": "^4.5.1",
+        "csv": "^6.4.1",
         "dotenv": "^17.2.4",
         "drizzle-orm": "^1.0.0-beta.15-859cf75",
         "pg": "^8.20.0",
@@ -120,7 +121,6 @@
       "resolved": "https://registry.npmjs.org/@azure-rest/core-client/-/core-client-2.5.1.tgz",
       "integrity": "sha512-EHaOXW0RYDKS5CFffnixdyRPak5ytiCtU7uXDcP/uiY+A6jFRwNGzzJBiznkCzvi5EYpY+YWinieqHb0oY916A==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@azure/abort-controller": "^2.1.2",
         "@azure/core-auth": "^1.10.0",
@@ -138,7 +138,6 @@
       "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz",
       "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "tslib": "^2.6.2"
       },
@@ -151,7 +150,6 @@
       "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.10.1.tgz",
       "integrity": "sha512-ykRMW8PjVAn+RS6ww5cmK9U2CyH9p4Q88YJwvUslfuMmN98w/2rdGRLPqJYObapBCdzBVeDgYWdJnFPFb7qzpg==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@azure/abort-controller": "^2.1.2",
         "@azure/core-util": "^1.13.0",
@@ -185,7 +183,6 @@
       "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.3.2.tgz",
       "integrity": "sha512-Tf6ltdKzOJEgxZeWLCjMxrxbodB/ZeCbzzA1A2qHbhzAjzjHoBVSUeSl/baT/oHAxhc4qdqVaDKnc2+iE932gw==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@azure/abort-controller": "^2.1.2"
       },
@@ -202,7 +199,6 @@
       "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.7.2.tgz",
       "integrity": "sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@azure/abort-controller": "^2.0.0",
         "@azure/core-util": "^1.2.0",
@@ -218,7 +214,6 @@
       "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.6.2.tgz",
       "integrity": "sha512-YKWi9YuCU04B55h25cnOYZHxXYtEvQEbKST5vqRga7hWY9ydd3FZHdeQF8pyh+acWZvppw13M/LMGx0LABUVMA==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "tslib": "^2.6.2"
       },
@@ -250,7 +245,6 @@
       "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.3.1.tgz",
       "integrity": "sha512-9MWKevR7Hz8kNzzPLfX4EAtGM2b8mr50HPDBvio96bURP/9C+HjdH3sBlLSNNrvRAr5/k/svoH457gB5IKpmwQ==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "tslib": "^2.6.2"
       },
@@ -263,7 +257,6 @@
       "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.13.1.tgz",
       "integrity": "sha512-XPArKLzsvl0Hf0CaGyKHUyVgF7oDnhKoP85Xv6M4StF/1AhfORhZudHtOyf2s+FcbuQ9dPRAjB8J2KvRRMUK2A==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@azure/abort-controller": "^2.1.2",
         "@typespec/ts-http-runtime": "^0.3.0",
@@ -301,7 +294,6 @@
       "resolved": "https://registry.npmjs.org/@azure/keyvault-common/-/keyvault-common-2.0.0.tgz",
       "integrity": "sha512-wRLVaroQtOqfg60cxkzUkGKrKMsCP6uYXAOomOIysSMyt1/YM0eUn9LqieAWM8DLcU4+07Fio2YGpPeqUbpP9w==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@azure/abort-controller": "^2.0.0",
         "@azure/core-auth": "^1.3.0",
@@ -321,7 +313,6 @@
       "resolved": "https://registry.npmjs.org/@azure/keyvault-keys/-/keyvault-keys-4.10.0.tgz",
       "integrity": "sha512-eDT7iXoBTRZ2n3fLiftuGJFD+yjkiB1GNqzU2KbY1TLYeXeSPVTVgn2eJ5vmRTZ11978jy2Kg2wI7xa9Tyr8ag==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@azure-rest/core-client": "^2.3.3",
         "@azure/abort-controller": "^2.1.2",
@@ -345,7 +336,6 @@
       "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.3.0.tgz",
       "integrity": "sha512-fCqPIfOcLE+CGqGPd66c8bZpwAji98tZ4JI9i/mlTNTlsIWslCfpg48s/ypyLxZTump5sypjrKn2/kY7q8oAbA==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@typespec/ts-http-runtime": "^0.3.0",
         "tslib": "^2.6.2"
@@ -359,7 +349,6 @@
       "resolved": "https://registry.npmjs.org/@azure/msal-browser/-/msal-browser-4.29.0.tgz",
       "integrity": "sha512-/f3eHkSNUTl6DLQHm+bKecjBKcRQxbd/XLx8lvSYp8Nl/HRyPuIPOijt9Dt0sH50/SxOwQ62RnFCmFlGK+bR/w==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@azure/msal-common": "15.15.0"
       },
@@ -372,7 +361,6 @@
       "resolved": "https://registry.npmjs.org/@azure/msal-common/-/msal-common-15.15.0.tgz",
       "integrity": "sha512-/n+bN0AKlVa+AOcETkJSKj38+bvFs78BaP4rNtv3MJCmPH0YrHiskMRe74OhyZ5DZjGISlFyxqvf9/4QVEi2tw==",
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=0.8.0"
       }
@@ -382,7 +370,6 @@
       "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-3.8.8.tgz",
       "integrity": "sha512-+f1VrJH1iI517t4zgmuhqORja0bL6LDQXfBqkjuMmfTYXTQQnh1EvwwxO3UbKLT05N0obF72SRHFrC1RBDv5Gg==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@azure/msal-common": "15.15.0",
         "jsonwebtoken": "^9.0.0",
@@ -1473,8 +1460,7 @@
       "version": "5.7.0",
       "resolved": "https://registry.npmjs.org/@js-joda/core/-/core-5.7.0.tgz",
       "integrity": "sha512-WBu4ULVVxySLLzK1Ppq+OdfP+adRS4ntmDQT915rzDJ++i95gc2jZkM5B6LWEAwN3lGXpfie3yPABozdD3K3Vg==",
-      "license": "BSD-3-Clause",
+      "license": "BSD-3-Clause"
-      "peer": true
     },
     "node_modules/@js-temporal/polyfill": {
       "version": "0.5.1",
@@ -1802,6 +1788,7 @@
       "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.8.tgz",
       "integrity": "sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==",
       "license": "MIT",
+      "peer": true,
       "funding": {
         "type": "opencollective",
         "url": "https://opencollective.com/popperjs"
@@ -2247,8 +2234,7 @@
       "version": "0.5.0",
       "resolved": "https://registry.npmjs.org/@tediousjs/connection-string/-/connection-string-0.5.0.tgz",
       "integrity": "sha512-7qSgZbincDDDFyRweCIEvZULFAw5iz/DeunhvuxpL31nfntX3P4Yd4HkHBRg9H8CdqY1e5WFN1PZIz/REL9MVQ==",
-      "license": "MIT",
+      "license": "MIT"
-      "peer": true
     },
     "node_modules/@types/bootstrap": {
       "version": "5.2.10",
@@ -2325,6 +2311,7 @@
       "resolved": "https://registry.npmjs.org/@types/node/-/node-25.4.0.tgz",
       "integrity": "sha512-9wLpoeWuBlcbBpOY3XmzSTG3oscB6xjBEEtn+pYXTfhyXhIxC5FsBer2KTopBlvKEiW9l13po9fq+SJY/5lkhw==",
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "undici-types": "~7.18.0"
       }
@@ -2335,6 +2322,7 @@
       "integrity": "sha512-gT+oueVQkqnj6ajGJXblFR4iavIXWsGAFCk3dP4Kki5+a9R4NMt0JARdk6s8cUKcfUoqP5dAtDSLU8xYUTFV+Q==",
       "devOptional": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@types/node": "*",
         "pg-protocol": "*",
@@ -2346,7 +2334,6 @@
       "resolved": "https://registry.npmjs.org/@types/readable-stream/-/readable-stream-4.0.23.tgz",
       "integrity": "sha512-wwXrtQvbMHxCbBgjHaMGEmImFTQxxpfMOR/ZoQnXxB1woqkUbdLGFDgauo00Py9IudiaqSeiBiulSV9i6XIPig==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@types/node": "*"
       }
@@ -2362,7 +2349,6 @@
       "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.4.tgz",
       "integrity": "sha512-CI0NhTrz4EBaa0U+HaaUZrJhPoso8sG7ZFya8uQoBA57fjzrjRSv87ekCjLZOFExN+gXE/z0xuN2QfH4H2HrLQ==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "http-proxy-agent": "^7.0.0",
         "https-proxy-agent": "^7.0.0",
@@ -2383,7 +2369,6 @@
       "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
       "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "event-target-shim": "^5.0.0"
       },
@@ -2408,7 +2393,6 @@
       "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz",
       "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==",
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">= 14"
       }
@@ -2542,6 +2526,7 @@
       "resolved": "https://registry.npmjs.org/astro/-/astro-5.18.1.tgz",
       "integrity": "sha512-m4VWilWZ+Xt6NPoYzC4CgGZim/zQUO7WFL0RHCH0AiEavF1153iC3+me2atDvXpf/yX4PyGUeD8wZLq1cirT3g==",
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@astrojs/compiler": "^2.13.0",
         "@astrojs/internal-helpers": "0.7.6",
@@ -2683,15 +2668,13 @@
           "url": "https://feross.org/support"
         }
       ],
-      "license": "MIT",
+      "license": "MIT"
-      "peer": true
     },
     "node_modules/bl": {
       "version": "6.1.6",
       "resolved": "https://registry.npmjs.org/bl/-/bl-6.1.6.tgz",
       "integrity": "sha512-jLsPgN/YSvPUg9UX0Kd73CXpm2Psg9FxMeCSXnk3WBO3CMT10JMwijubhGfHCnFu6TPn1ei3b975dxv7K2pWVg==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@types/readable-stream": "^4.0.0",
         "buffer": "^6.0.3",
@@ -2765,7 +2748,6 @@
         }
       ],
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "base64-js": "^1.3.1",
         "ieee754": "^1.2.1"
@@ -2775,15 +2757,13 @@
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
       "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==",
-      "license": "BSD-3-Clause",
+      "license": "BSD-3-Clause"
-      "peer": true
     },
     "node_modules/bundle-name": {
       "version": "4.1.0",
       "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz",
       "integrity": "sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "run-applescript": "^7.0.0"
       },
@@ -3091,6 +3071,39 @@
       "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==",
       "license": "MIT"
     },
+    "node_modules/csv": {
+      "version": "6.4.1",
+      "resolved": "https://registry.npmjs.org/csv/-/csv-6.4.1.tgz",
+      "integrity": "sha512-ajGosmTGnTwYyGl8STqZDu7R6LkDf3xL39XiOmliV/GufQeVUxHzTKIm4NOBCwmEuujK7B6isxs4Uqt9GcRCvA==",
+      "license": "MIT",
+      "dependencies": {
+        "csv-generate": "^4.5.0",
+        "csv-parse": "^6.1.0",
+        "csv-stringify": "^6.6.0",
+        "stream-transform": "^3.4.0"
+      },
+      "engines": {
+        "node": ">= 0.1.90"
+      }
+    },
+    "node_modules/csv-generate": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.5.0.tgz",
+      "integrity": "sha512-aQr/vmOKyBSBHNwYhAoXw1+kUsPnMSwmYgpNoo36rIXoG1ecWILnvPGZeQ6oUjzrWknZAD3+jfpqYOBAl4x15A==",
+      "license": "MIT"
+    },
+    "node_modules/csv-parse": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-6.1.0.tgz",
+      "integrity": "sha512-CEE+jwpgLn+MmtCpVcPtiCZpVtB6Z2OKPTr34pycYYoL7sxdOkXDdQ4lRiw6ioC0q6BLqhc6cKweCVvral8yhw==",
+      "license": "MIT"
+    },
+    "node_modules/csv-stringify": {
+      "version": "6.6.0",
+      "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.6.0.tgz",
+      "integrity": "sha512-YW32lKOmIBgbxtu3g5SaiqWNwa/9ISQt2EcgOq0+RAIFufFp9is6tqNnKahqE5kuKvrnYAzs28r+s6pXJR8Vcw==",
+      "license": "MIT"
+    },
     "node_modules/debug": {
       "version": "4.4.3",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
@@ -3126,7 +3139,6 @@
       "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.5.0.tgz",
       "integrity": "sha512-H9LMLr5zwIbSxrmvikGuI/5KGhZ8E2zH3stkMgM5LpOWDutGM2JZaj460Udnf1a+946zc7YBgrqEWwbk7zHvGw==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "bundle-name": "^4.1.0",
         "default-browser-id": "^5.0.0"
@@ -3143,7 +3155,6 @@
       "resolved": "https://registry.npmjs.org/default-browser-id/-/default-browser-id-5.0.1.tgz",
       "integrity": "sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q==",
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=18"
       },
@@ -3156,7 +3167,6 @@
       "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz",
       "integrity": "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==",
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=12"
       },
@@ -4009,7 +4019,6 @@
       "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz",
       "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==",
       "license": "Apache-2.0",
-      "peer": true,
       "dependencies": {
         "safe-buffer": "^5.0.1"
       }
@@ -4180,7 +4189,6 @@
       "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
       "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=6"
       }
@@ -4196,7 +4204,6 @@
       "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
       "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=0.8.x"
       }
@@ -4715,7 +4722,6 @@
       "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz",
       "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "agent-base": "^7.1.0",
         "debug": "^4.3.4"
@@ -4729,7 +4735,6 @@
       "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz",
       "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "agent-base": "^7.1.2",
         "debug": "4"
@@ -4743,7 +4748,6 @@
       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz",
       "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "safer-buffer": ">= 2.1.2 < 3.0.0"
       },
@@ -4773,8 +4777,7 @@
           "url": "https://feross.org/support"
         }
       ],
-      "license": "BSD-3-Clause",
+      "license": "BSD-3-Clause"
-      "peer": true
     },
     "node_modules/immutable": {
       "version": "5.1.5",
@@ -4912,8 +4915,7 @@
       "version": "0.3.2",
       "resolved": "https://registry.npmjs.org/js-md4/-/js-md4-0.3.2.tgz",
       "integrity": "sha512-/GDnfQYsltsjRswQhN9fhv3EMw2sCpUdrdxyWDOUK7eyD++r3gRhzgiQgc/x4MAv2i1iuQ4lxO5mvqM3vj4bwA==",
-      "license": "MIT",
+      "license": "MIT"
-      "peer": true
     },
     "node_modules/js-yaml": {
       "version": "4.1.1",
@@ -4939,7 +4941,6 @@
       "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.3.tgz",
       "integrity": "sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "jws": "^4.0.1",
         "lodash.includes": "^4.3.0",
@@ -4962,7 +4963,6 @@
       "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz",
       "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "buffer-equal-constant-time": "^1.0.1",
         "ecdsa-sig-formatter": "1.0.11",
@@ -4974,7 +4974,6 @@
       "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz",
       "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "jwa": "^2.0.1",
         "safe-buffer": "^5.0.1"
@@ -4993,50 +4992,43 @@
       "version": "4.3.0",
       "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
       "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==",
-      "license": "MIT",
+      "license": "MIT"
-      "peer": true
     },
     "node_modules/lodash.isboolean": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
       "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==",
-      "license": "MIT",
+      "license": "MIT"
-      "peer": true
     },
     "node_modules/lodash.isinteger": {
       "version": "4.0.4",
       "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz",
       "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==",
-      "license": "MIT",
+      "license": "MIT"
-      "peer": true
     },
     "node_modules/lodash.isnumber": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz",
       "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==",
-      "license": "MIT",
+      "license": "MIT"
-      "peer": true
     },
     "node_modules/lodash.isplainobject": {
       "version": "4.0.6",
       "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz",
       "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==",
-      "license": "MIT",
+      "license": "MIT"
-      "peer": true
     },
     "node_modules/lodash.isstring": {
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz",
       "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==",
-      "license": "MIT",
+      "license": "MIT"
-      "peer": true
     },
     "node_modules/lodash.once": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz",
       "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==",
-      "license": "MIT",
+      "license": "MIT"
-      "peer": true
     },
     "node_modules/loglevel": {
       "version": "1.9.2",
@@ -5948,7 +5940,6 @@
       "resolved": "https://registry.npmjs.org/mssql/-/mssql-11.0.1.tgz",
       "integrity": "sha512-KlGNsugoT90enKlR8/G36H0kTxPthDhmtNUCwEHvgRza5Cjpjoj+P2X6eMpFUDN7pFrJZsKadL4x990G8RBE1w==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@tediousjs/connection-string": "^0.5.0",
         "commander": "^11.0.0",
@@ -5969,7 +5960,6 @@
       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
       "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "safer-buffer": ">= 2.1.2 < 3.0.0"
       },
@@ -5982,7 +5972,6 @@
       "resolved": "https://registry.npmjs.org/tedious/-/tedious-18.6.2.tgz",
       "integrity": "sha512-g7jC56o3MzLkE3lHkaFe2ZdOVFBahq5bsB60/M4NYUbocw/MCrS89IOEQUFr+ba6pb8ZHczZ/VqCyYeYq0xBAg==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@azure/core-auth": "^1.7.2",
         "@azure/identity": "^4.2.1",
@@ -6036,8 +6025,7 @@
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/native-duplexpair/-/native-duplexpair-1.0.0.tgz",
       "integrity": "sha512-E7QQoM+3jvNtlmyfqRZ0/U75VFgCls+fSkbml2MpgWkWyz3ox8Y58gNhfuziuQYGNNQAbFZJQck55LHCnCK6CA==",
-      "license": "MIT",
+      "license": "MIT"
-      "peer": true
     },
     "node_modules/neotraverse": {
       "version": "0.6.18",
@@ -6152,7 +6140,6 @@
       "resolved": "https://registry.npmjs.org/open/-/open-10.2.0.tgz",
       "integrity": "sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "default-browser": "^5.2.1",
         "define-lazy-prop": "^3.0.0",
@@ -6250,6 +6237,7 @@
       "resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz",
       "integrity": "sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==",
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "pg-connection-string": "^2.12.0",
         "pg-pool": "^3.13.0",
@@ -6457,7 +6445,6 @@
       "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
       "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==",
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">= 0.6.0"
       }
@@ -6521,7 +6508,6 @@
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz",
       "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "abort-controller": "^3.0.0",
         "buffer": "^6.0.3",
@@ -6787,14 +6773,14 @@
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.4.1.tgz",
       "integrity": "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==",
-      "license": "MIT",
+      "license": "MIT"
-      "peer": true
     },
     "node_modules/rollup": {
       "version": "4.59.0",
       "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.59.0.tgz",
       "integrity": "sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg==",
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@types/estree": "1.0.8"
       },
@@ -6839,7 +6825,6 @@
       "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.1.0.tgz",
       "integrity": "sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==",
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=18"
       },
@@ -6865,21 +6850,20 @@
           "url": "https://feross.org/support"
         }
       ],
-      "license": "MIT",
+      "license": "MIT"
-      "peer": true
     },
     "node_modules/safer-buffer": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
-      "license": "MIT",
+      "license": "MIT"
-      "peer": true
     },
     "node_modules/sass": {
       "version": "1.98.0",
       "resolved": "https://registry.npmjs.org/sass/-/sass-1.98.0.tgz",
       "integrity": "sha512-+4N/u9dZ4PrgzGgPlKnaaRQx64RO0JBKs9sDhQ2pLgN6JQZ25uPQZKQYaBJU48Kd5BxgXoJ4e09Dq7nMcOUW3A==",
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "chokidar": "^4.0.0",
         "immutable": "^5.1.5",
@@ -7065,8 +7049,7 @@
       "version": "1.1.3",
       "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz",
       "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==",
-      "license": "BSD-3-Clause",
+      "license": "BSD-3-Clause"
-      "peer": true
     },
     "node_modules/standardwebhooks": {
       "version": "1.0.0",
@@ -7093,12 +7076,17 @@
       "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==",
       "license": "MIT"
     },
+    "node_modules/stream-transform": {
+      "version": "3.4.0",
+      "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.4.0.tgz",
+      "integrity": "sha512-QO3OGhKyeIV8p6eRQdG+W6WounFw519zk690hHCNfhgfP9bylVS+NTXsuBc7n+RsGn31UgFPGrWYIgoAbArKEw==",
+      "license": "MIT"
+    },
     "node_modules/string_decoder": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
       "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "safe-buffer": "~5.2.0"
       }
@@ -7192,7 +7180,6 @@
       "resolved": "https://registry.npmjs.org/tarn/-/tarn-3.0.2.tgz",
       "integrity": "sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==",
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=8.0.0"
       }
@@ -7202,7 +7189,6 @@
       "resolved": "https://registry.npmjs.org/tedious/-/tedious-19.2.1.tgz",
       "integrity": "sha512-pk1Q16Yl62iocuQB+RWbg6rFUFkIyzqOFQ6NfysCltRvQqKwfurgj8v/f2X+CKvDhSL4IJ0cCOfCHDg9PWEEYA==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@azure/core-auth": "^1.7.2",
         "@azure/identity": "^4.2.1",
@@ -7311,6 +7297,7 @@
       "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==",
       "devOptional": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "esbuild": "~0.27.0",
         "get-tsconfig": "^4.7.5"
@@ -7342,6 +7329,7 @@
       "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
       "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
       "license": "Apache-2.0",
+      "peer": true,
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"
@@ -7682,7 +7670,6 @@
       "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
       "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
       "license": "MIT",
-      "peer": true,
       "bin": {
         "uuid": "dist/bin/uuid"
       }
@@ -8335,7 +8322,6 @@
       "resolved": "https://registry.npmjs.org/wsl-utils/-/wsl-utils-0.1.0.tgz",
       "integrity": "sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "is-wsl": "^3.1.0"
       },
@@ -8414,6 +8400,7 @@
       "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz",
       "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==",
       "license": "MIT",
+      "peer": true,
       "funding": {
         "url": "https://github.com/sponsors/colinhacks"
       }

package.json

@@ -17,6 +17,7 @@
     "bootstrap": "^5.3.8",
     "chalk": "^5.6.2",
     "chart.js": "^4.5.1",
+    "csv": "^6.4.1",
     "dotenv": "^17.2.4",
     "drizzle-orm": "^1.0.0-beta.15-859cf75",
     "pg": "^8.20.0",

scripts/csvprices.ts

@@ -0,0 +1,87 @@
+import 'dotenv/config';
+import { db, ClosePool } from '../src/db/index.ts';
+import chalk from 'chalk';
+import fs from "fs";
+//import path from "node:path";
+import { parse, stringify, transform } from 'csv';
+import { client } from '../src/db/typesense.ts';
+
+async function PricesFromCSV() {
+
+  const inputFilePath = 'scripts/test.tcgcollector.csv';
+  const outputFilePath = 'scripts/output.csv';
+
+  // Create read and write streams
+  const inputStream = fs.createReadStream(inputFilePath, 'utf8');
+  const outputStream = fs.createWriteStream(outputFilePath);
+
+  // Define the transformation logic
+  const transformer = transform({ parallel: 1 }, async function(this: any, row: any, callback: any) {
+    try {
+      // Specific query bsaed on tcgcollector CSV
+      const query = String(Object.values(row)[1]);
+      const setname = String(Object.values(row)[4]).replace(/Wizards of the coast promos/ig,'WoTC Promo');
+      const cardNumber = String(Object.values(row)[7]);
+      console.log(`${query} ${cardNumber} : ${setname}`);
+
+      // Use Typesense to find the card because we can easily use the combined fields
+      let cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `setName:\`${setname}\` && number:${cardNumber}` });
+      if (cards.hits?.length === 0) {
+        // Try without card number
+        cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `setName:\`${setname}\`` });
+      }
+      if (cards.hits?.length === 0) {
+        // Try without set name
+        cards = await client.collections('cards').documents().search({ q: query, query_by: 'productName', filter_by: `number:${cardNumber}` });
+      }
+      if (cards.hits?.length === 0) {
+        // I give up, just output the values from the csv
+        console.log(chalk.red('  - not found'));
+        const newRow = { ...row };
+        newRow.Variant = '';
+        newRow.marketPrice = '';
+        this.push(newRow);
+      }
+      else {
+        for (const card of cards.hits?.map((hit: any) => hit.document) ?? []) {
+          console.log(chalk.blue(`  - ${card.cardId} : ${card.productName} : ${card.number}`), chalk.yellow(`${card.setName}`), chalk.green(`${card.variant}`));
+          const variant = await db.query.cards.findFirst({
+            with: { prices: true, tcgdata: true },
+            where: { cardId: card.cardId }
+          });
+          const newRow = { ...row };
+          newRow.Variant = variant?.variant;
+          newRow.marketPrice = variant?.prices.find(p => p.condition === 'Near Mint')?.marketPrice;
+          this.push(newRow);
+        }
+      }
+
+      callback();
+    } catch (error) {
+      callback(error);
+    }
+  });
+
+  // Pipe the streams: Read -> Parse -> Transform -> Stringify -> Write
+  inputStream
+    .on('error', (error) => console.error('Input stream error:', error))
+    .pipe(parse({ columns: true, trim: true }))
+    .on('error', (error) => console.error('Parse error:', error))
+    .pipe(transformer)
+    .on('error', (error) => console.error('Transform error:', error))
+    .pipe(stringify({ header: true }))
+    .on('error', (error) => console.error('Stringify error:', error))
+    .pipe(outputStream);
+
+  outputStream.on('finish', () => {
+    console.log(`Successfully written to ${outputFilePath}`);
+    ClosePool();
+  });
+
+  outputStream.on('error', (error) => {
+    console.error('An error occurred in the process:', error);
+    ClosePool();
+  });
+}
+
+await PricesFromCSV();

src/db/relations.ts

@@ -8,12 +8,19 @@ export const relations = defineRelations(schema, (r) => ({
             to: r.skus.skuId,
         }),
     },
+    salesHistory: {
+        sku: r.one.skus({
+            from: r.salesHistory.skuId,
+            to: r.skus.skuId,
+        }),
+    },
     skus: {
         card: r.one.cards({
             from: [r.skus.productId, r.skus.variant],
             to: [r.cards.productId, r.cards.variant],
         }),
         history: r.many.priceHistory(),
+        latestSales: r.many.salesHistory(),
     },
     cards: {
         prices: r.many.skus(),

src/db/schema.ts

@@ -109,6 +109,21 @@ export const priceHistory = pokeSchema.table('price_history', {
 	primaryKey({ name: 'pk_price_history', columns: [table.skuId, table.calculatedAt] })
 ]);
 
+export const salesHistory = pokeSchema.table('sales_history',{
+	skuId: integer().notNull(),
+	orderDate: timestamp().notNull(),
+	title: varchar({ length: 255 }),
+	customListingId: varchar({ length: 255 }),
+	language: varchar({ length: 100 }),
+	listingType: varchar({ length: 100 }),
+	purchasePrice: decimal({ precision: 10, scale: 2 }),
+	quantity: integer(),
+	shippingPrice: decimal({ precision: 10, scale: 2 })
+},
+(table) => [
+	primaryKey({ name: 'pk_sales_history', columns: [table.skuId, table.orderDate] })
+]);
+
 export const processingSkus = pokeSchema.table('processing_skus', {
 	skuId: integer().primaryKey(),
 });