diff --git a/docker-compose.yaml b/docker-compose.yaml index c29bb8b..4af78d9 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -25,6 +25,15 @@ services: volumes: - minio_data:/data + redis: + image: redis:7-alpine + container_name: pandektes-redis + ports: + - "6379:6379" + volumes: + - redis_data:/data + volumes: postgres_data: - minio_data: \ No newline at end of file + minio_data: + redis_data: \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 03fde30..f487519 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "@langchain/core": "^1.1.29", "@langchain/google-genai": "^2.1.22", "@nestjs/apollo": "^13.2.4", + "@nestjs/bullmq": "^11.0.4", "@nestjs/common": "^11.1.14", "@nestjs/config": "^4.0.3", "@nestjs/core": "^11.1.14", @@ -24,6 +25,7 @@ "@nestjs/platform-express": "^11.0.1", "@prisma/adapter-pg": "^7.4.2", "@prisma/client": "^7.4.2", + "bullmq": "^5.70.1", "cheerio": "^1.2.0", "class-transformer": "^0.5.1", "class-validator": "^0.15.1", @@ -2676,6 +2678,12 @@ } } }, + "node_modules/@ioredis/commands": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.5.0.tgz", + "integrity": "sha512-eUgLqrMf8nJkZxT24JvVRrQya1vZkQh8BBeYNwGDqa5I0VUi8ACx7uFvAaLxintokpTenkK6DASvo/bvNbBGow==", + "license": "MIT" + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -3455,6 +3463,84 @@ "node": ">=16" } }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", + "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", + "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", + "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", + "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", + "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", + "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@napi-rs/canvas": { "version": "0.1.80", "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.80.tgz", @@ -3688,6 +3774,34 @@ } } }, + "node_modules/@nestjs/bull-shared": { + "version": "11.0.4", + "resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-11.0.4.tgz", + "integrity": "sha512-VBJcDHSAzxQnpcDfA0kt9MTGUD1XZzfByV70su0W0eDCQ9aqIEBlzWRW21tv9FG9dIut22ysgDidshdjlnczLw==", + "license": "MIT", + "dependencies": { + "tslib": "2.8.1" + }, + "peerDependencies": { + "@nestjs/common": "^10.0.0 || ^11.0.0", + "@nestjs/core": "^10.0.0 || ^11.0.0" + } + }, + "node_modules/@nestjs/bullmq": { + "version": "11.0.4", + "resolved": "https://registry.npmjs.org/@nestjs/bullmq/-/bullmq-11.0.4.tgz", + "integrity": "sha512-wBzK9raAVG0/6NTMdvLGM4/FQ1lsB35/pYS8L6a0SDgkTiLpd7mAjQ8R692oMx5s7IjvgntaZOuTUrKYLNfIkA==", + "license": "MIT", + "dependencies": { + "@nestjs/bull-shared": "^11.0.4", + "tslib": "2.8.1" + }, + "peerDependencies": { + "@nestjs/common": "^10.0.0 || ^11.0.0", + "@nestjs/core": "^10.0.0 || ^11.0.0", + "bullmq": "^3.0.0 || ^4.0.0 || ^5.0.0" + } + }, "node_modules/@nestjs/cli": { "version": "11.0.16", "resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-11.0.16.tgz", @@ -7103,6 +7217,21 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "license": "MIT" }, + "node_modules/bullmq": { + "version": "5.70.1", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.70.1.tgz", + "integrity": "sha512-HjfGHfICkAClrFL0Y07qNbWcmiOCv1l+nusupXUjrvTPuDEyPEJ23MP0lUwUs/QEy1a3pWt/P/sCsSZ1RjRK+w==", + "license": "MIT", + "dependencies": { + "cron-parser": "4.9.0", + "ioredis": "5.9.3", + "msgpackr": "1.11.5", + "node-abort-controller": "3.1.1", + "semver": "7.7.4", + "tslib": "2.8.1", + "uuid": "11.1.0" + } + }, "node_modules/busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -7501,6 +7630,15 @@ "node": ">=0.8" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -7730,6 +7868,18 @@ "dev": true, "license": "MIT" }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "license": "MIT", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/cross-inspect": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cross-inspect/-/cross-inspect-1.0.1.tgz", @@ -7918,7 +8068,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", - "devOptional": true, "license": "Apache-2.0", "engines": { "node": ">=0.10" @@ -7940,6 +8089,16 @@ "devOptional": true, "license": "MIT" }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=8" + } + }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -9702,6 +9861,30 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "license": "ISC" }, + "node_modules/ioredis": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.9.3.tgz", + "integrity": "sha512-VI5tMCdeoxZWU5vjHWsiE/Su76JGhBvWF1MJnV9ZtGltHk9BmD48oDq8Tj8haZ85aceXZMxLNDQZRVo5QKNgXA==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "1.5.0", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -10988,6 +11171,18 @@ "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", "license": "MIT" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", + "license": "MIT" + }, "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -11078,6 +11273,15 @@ "url": "https://github.com/sponsors/wellwelwel" } }, + "node_modules/luxon": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", + "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/magic-string": { "version": "0.30.17", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.17.tgz", @@ -11313,6 +11517,37 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/msgpackr": { + "version": "1.11.5", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.5.tgz", + "integrity": "sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA==", + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", + "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" + } + }, "node_modules/mustache": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", @@ -11409,7 +11644,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", - "dev": true, "license": "MIT" }, "node_modules/node-emoji": { @@ -11429,6 +11663,21 @@ "devOptional": true, "license": "MIT" }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "license": "MIT", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -12490,6 +12739,27 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/reflect-metadata": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", @@ -13028,6 +13298,12 @@ "node": ">=8" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", diff --git a/package.json b/package.json index 8d6d80e..d8a958a 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "@langchain/core": "^1.1.29", "@langchain/google-genai": "^2.1.22", "@nestjs/apollo": "^13.2.4", + "@nestjs/bullmq": "^11.0.4", "@nestjs/common": "^11.1.14", "@nestjs/config": "^4.0.3", "@nestjs/core": "^11.1.14", @@ -35,6 +36,7 @@ "@nestjs/platform-express": "^11.0.1", "@prisma/adapter-pg": "^7.4.2", "@prisma/client": "^7.4.2", + "bullmq": "^5.70.1", "cheerio": "^1.2.0", "class-transformer": "^0.5.1", "class-validator": "^0.15.1", diff --git a/src/cases/processors/case-queue.listener.ts b/src/cases/processors/case-queue.listener.ts new file mode 100644 index 0000000..ebd7ae0 --- /dev/null +++ b/src/cases/processors/case-queue.listener.ts @@ -0,0 +1,113 @@ +import { QueueEventsListener, QueueEventsHost, OnQueueEvent, InjectQueue } from '@nestjs/bullmq'; +import { Queue } from 'bullmq'; +import { PrismaService } from '../../common/prisma/prisma.service'; +import { CaseStatus } from '@prisma/client'; +import { Logger } from '@nestjs/common'; + +@QueueEventsListener('case-processing') +export class CaseQueueListener extends QueueEventsHost { + private readonly logger = new Logger(CaseQueueListener.name); + + constructor( + private prisma: PrismaService, + @InjectQueue('case-processing') private readonly queue: Queue, + ) { + super(); + } + + @OnQueueEvent('active') + async onActive({ jobId }: { jobId: string }) { + try { + const job = await this.queue.getJob(jobId); + if (!job) return; + + const { caseId } = job.data; + await this.prisma.caseLaw.update({ + where: { id: caseId }, + data: { status: CaseStatus.PROCESSING }, + }); + } catch (error) { + this.logger.error(`Critical Exception in onActive for Job ${jobId}: ${error.message}`); + } + } + + @OnQueueEvent('progress') + async onProgress({ jobId, data }: { jobId: string; data: any }) { + try { + const job = await this.queue.getJob(jobId); + if (!job || typeof data !== 'string') return; + + const { caseId } = job.data; + await this.prisma.caseLaw.update({ + where: { id: caseId }, + data: { + logs: { + push: data, + }, + }, + }); + } catch (error) { + this.logger.error(`Progress update failed for Job ${jobId}: ${error.message}`); + } + } + + @OnQueueEvent('completed') + async onCompleted({ jobId, returnvalue }: { jobId: string; returnvalue: any }) { + try { + const job = await this.queue.getJob(jobId); + if (!job) return; + + const { caseId } = job.data; + if (!returnvalue) { + this.logger.error(`No return value from worker for Job ${jobId}`); + return; + } + + const { decisionDateAsDate, ...data } = returnvalue; + + await this.prisma.caseLaw.update({ + where: { id: caseId }, + data: { + ...data, + decisionDate: decisionDateAsDate ? new Date(decisionDateAsDate) : undefined, + status: CaseStatus.COMPLETED, + }, + }); + } catch (error) { + try { + const job = await this.queue.getJob(jobId); + if (job) { + await this.prisma.caseLaw.update({ + where: { id: job.data.caseId }, + data: { + status: CaseStatus.FAILED, + processingError: `Sync Error: ${error.message}` + } + }); + } + } catch (innerError) { + this.logger.error(`Could not mark as failed: ${innerError.message}`); + } + } + } + + @OnQueueEvent('failed') + async onFailed({ jobId, failedReason }: { jobId: string; failedReason: string }) { + try { + const job = await this.queue.getJob(jobId); + if (!job) return; + + const { caseId } = job.data; + + await this.prisma.caseLaw.update({ + where: { id: caseId }, + data: { + status: CaseStatus.FAILED, + processingError: failedReason, + }, + }); + } catch (error) { + this.logger.error(`Critical Exception in onFailed for Job ${jobId}: ${error.message}`); + } + } +}