fix: harden result polling and Redis 8.10 operations
Preserve tournament result jobs through lock contention and transient failures, atomically schedule retries before acknowledgement, and publish initial polls after releasing the preparation lock. Fix Redis connection blocking, durability Hash cleanup, duplicate auction request ordering, and scheduled-job promotion. Use native Redis 8.10 stream, lock, counter, and field-expiring cache operations with invalidation fencing. Repair 90 test type errors and add a repeatable API typecheck command. Validation: 704 API tests, 129 web tests, 31 real Redis integration tests, 3 dependency checks, workspace type checks and builds, and changed-file lint.
This commit is contained in:
1 parent
0c4c4239ec
commit
4b8ccf0077
49 files changed
+2214
-371
No files matched your search
@@ -1,12 +1,17 @@
|
||||
import mongoose from 'mongoose';
|
||||
import { ConnectionService } from '../global/database/connection.service';
|
||||
import { DistributedLockService } from '../global/redis/distributed-lock.service';
|
||||
import { RedisService } from '../global/redis/redis.service';
|
||||
import { RedisService, StreamEntry } from '../global/redis/redis.service';
|
||||
import { LegacyImportService } from '../legacy-import/legacy-import.service';
|
||||
import { RiotAccountEnrichmentService } from '../riot/riot-account-enrichment.service';
|
||||
import { RiotRecentMatchesService } from '../riot/riot-recent-matches.service';
|
||||
import { TournamentResultService } from '../tournament-result/tournament-result.service';
|
||||
import { RIOT_JOB_GROUP, RiotRefreshSubJob } from './riot-job-queue.types';
|
||||
import {
|
||||
RIOT_JOB_GROUP,
|
||||
RIOT_JOB_HIGH_STREAM,
|
||||
RIOT_JOB_LOW_STREAM,
|
||||
RiotRefreshSubJob,
|
||||
} from './riot-job-queue.types';
|
||||
import { RiotJobQueueService } from './riot-job-queue.service';
|
||||
import { RiotJobWorkerService } from './riot-job-worker.service';
|
||||
import { RiotRefreshGateway } from './riot-refresh.gateway';
|
||||
@@ -16,9 +21,17 @@ function execResult<T>(value: T) {
|
||||
}
|
||||
|
||||
function makeWorker() {
|
||||
const readerClient = { isOpen: true, destroy: jest.fn() };
|
||||
const streamReader = {
|
||||
raw: jest.fn().mockReturnValue(readerClient),
|
||||
readGroup: jest.fn().mockResolvedValue(null),
|
||||
} as unknown as jest.Mocked<RedisService>;
|
||||
const redisService = {
|
||||
acknowledgeStream: jest.fn().mockResolvedValue(undefined),
|
||||
deleteStreamEntries: jest.fn().mockResolvedValue(undefined),
|
||||
acknowledgeAndDeleteStream: jest.fn().mockResolvedValue(undefined),
|
||||
createConsumerGroup: jest.fn().mockResolvedValue(undefined),
|
||||
createDedicatedConnection: jest.fn().mockResolvedValue(streamReader),
|
||||
raw: jest.fn(),
|
||||
readGroup: jest.fn(),
|
||||
} as unknown as jest.Mocked<RedisService>;
|
||||
const distributedLockService = {
|
||||
runWithRenewedLock: jest
|
||||
@@ -33,6 +46,7 @@ function makeWorker() {
|
||||
),
|
||||
} as unknown as jest.Mocked<DistributedLockService>;
|
||||
const jobQueueService = {
|
||||
promoteDueScheduledJobs: jest.fn().mockResolvedValue(0),
|
||||
markProcessing: jest.fn().mockResolvedValue(undefined),
|
||||
markCompleted: jest.fn().mockResolvedValue({
|
||||
parentJobId: 'parent-1',
|
||||
@@ -87,10 +101,106 @@ function makeWorker() {
|
||||
connectionService,
|
||||
refreshGateway,
|
||||
enrichmentService,
|
||||
streamReader,
|
||||
readerClient,
|
||||
};
|
||||
}
|
||||
|
||||
type WorkerInternals = {
|
||||
stopped: boolean;
|
||||
loop(): Promise<void>;
|
||||
tryBecomeLeader(): Promise<boolean>;
|
||||
readOne(
|
||||
stream: string,
|
||||
blockMs: number,
|
||||
): Promise<{ stream: string; message: StreamEntry } | null>;
|
||||
processEntry(stream: string, entry: StreamEntry): Promise<void>;
|
||||
};
|
||||
|
||||
describe('RiotJobWorkerService', () => {
|
||||
it('connects a dedicated reader after creating groups and only closes that reader', async () => {
|
||||
const { worker, redisService, streamReader, readerClient } = makeWorker();
|
||||
const internal = worker as unknown as WorkerInternals;
|
||||
jest.spyOn(internal, 'loop').mockResolvedValue(undefined);
|
||||
await worker.onModuleInit();
|
||||
expect(redisService.createConsumerGroup).toHaveBeenCalledWith(
|
||||
RIOT_JOB_HIGH_STREAM,
|
||||
RIOT_JOB_GROUP,
|
||||
);
|
||||
expect(redisService.createConsumerGroup).toHaveBeenCalledWith(
|
||||
RIOT_JOB_LOW_STREAM,
|
||||
RIOT_JOB_GROUP,
|
||||
);
|
||||
expect(
|
||||
redisService.createConsumerGroup.mock.invocationCallOrder[1],
|
||||
).toBeLessThan(
|
||||
redisService.createDedicatedConnection.mock.invocationCallOrder[0],
|
||||
);
|
||||
|
||||
await internal.readOne(RIOT_JOB_HIGH_STREAM, 100);
|
||||
expect(streamReader.readGroup).toHaveBeenCalledWith(
|
||||
RIOT_JOB_HIGH_STREAM,
|
||||
RIOT_JOB_GROUP,
|
||||
expect.any(String),
|
||||
'>',
|
||||
{ count: 1, blockMs: 100, claimMinIdleMs: 1800000 },
|
||||
);
|
||||
expect(redisService.readGroup).not.toHaveBeenCalled();
|
||||
|
||||
await worker.onModuleDestroy();
|
||||
expect(readerClient.destroy).toHaveBeenCalledTimes(1);
|
||||
expect(redisService.raw).not.toHaveBeenCalled();
|
||||
expect(internal.stopped).toBe(true);
|
||||
});
|
||||
|
||||
it.each([true, false])(
|
||||
'preserves high-before-low consumption (high available: %s)',
|
||||
async (highAvailable) => {
|
||||
const { worker, jobQueueService } = makeWorker();
|
||||
const internal = worker as unknown as WorkerInternals;
|
||||
const entry = { id: '1-0', message: { job: '{}' } };
|
||||
jest.spyOn(internal, 'tryBecomeLeader').mockResolvedValue(true);
|
||||
const read = jest
|
||||
.spyOn(internal, 'readOne')
|
||||
.mockImplementation(async (stream) => {
|
||||
if (stream === RIOT_JOB_HIGH_STREAM && !highAvailable) return null;
|
||||
return { stream, message: entry };
|
||||
});
|
||||
const process = jest
|
||||
.spyOn(internal, 'processEntry')
|
||||
.mockImplementation(async () => {
|
||||
internal.stopped = true;
|
||||
});
|
||||
await internal.loop();
|
||||
expect(jobQueueService.promoteDueScheduledJobs).toHaveBeenCalledTimes(1);
|
||||
expect(read.mock.calls).toEqual(
|
||||
highAvailable
|
||||
? [[RIOT_JOB_HIGH_STREAM, 100]]
|
||||
: [
|
||||
[RIOT_JOB_HIGH_STREAM, 100],
|
||||
[RIOT_JOB_LOW_STREAM, 1000],
|
||||
],
|
||||
);
|
||||
expect(process).toHaveBeenCalledWith(
|
||||
highAvailable ? RIOT_JOB_HIGH_STREAM : RIOT_JOB_LOW_STREAM,
|
||||
entry,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it('does not start processing a message returned during shutdown', async () => {
|
||||
const { worker } = makeWorker();
|
||||
const internal = worker as unknown as WorkerInternals;
|
||||
jest.spyOn(internal, 'tryBecomeLeader').mockResolvedValue(true);
|
||||
jest.spyOn(internal, 'readOne').mockImplementation(async (stream) => {
|
||||
await worker.onModuleDestroy();
|
||||
return { stream, message: { id: '1-0', message: {} } };
|
||||
});
|
||||
const process = jest.spyOn(internal, 'processEntry');
|
||||
await internal.loop();
|
||||
expect(process).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('acknowledges and deletes invalid stream entries', async () => {
|
||||
const { worker, redisService } = makeWorker();
|
||||
|
||||
@@ -106,15 +216,11 @@ describe('RiotJobWorkerService', () => {
|
||||
message: { job: 'not-json' },
|
||||
});
|
||||
|
||||
expect(redisService.acknowledgeStream).toHaveBeenCalledWith(
|
||||
expect(redisService.acknowledgeAndDeleteStream).toHaveBeenCalledWith(
|
||||
'riot:queue:high',
|
||||
RIOT_JOB_GROUP,
|
||||
'bad-1',
|
||||
);
|
||||
expect(redisService.deleteStreamEntries).toHaveBeenCalledWith(
|
||||
'riot:queue:high',
|
||||
'bad-1',
|
||||
);
|
||||
});
|
||||
|
||||
it('refreshes Riot identity and league entries while processing user history refresh jobs', async () => {
|
||||
@@ -165,19 +271,15 @@ describe('RiotJobWorkerService', () => {
|
||||
expect(
|
||||
enrichmentService.refreshRiotIdentity.mock.invocationCallOrder[0],
|
||||
).toBeLessThan(jobQueueService.markCompleted.mock.invocationCallOrder[0]);
|
||||
expect(redisService.acknowledgeStream).toHaveBeenCalledWith(
|
||||
expect(redisService.acknowledgeAndDeleteStream).toHaveBeenCalledWith(
|
||||
'riot:queue:high',
|
||||
RIOT_JOB_GROUP,
|
||||
'1-0',
|
||||
);
|
||||
expect(redisService.deleteStreamEntries).toHaveBeenCalledWith(
|
||||
'riot:queue:high',
|
||||
'1-0',
|
||||
);
|
||||
expect(
|
||||
redisService.acknowledgeStream.mock.invocationCallOrder[0],
|
||||
jobQueueService.markCompleted.mock.invocationCallOrder[0],
|
||||
).toBeLessThan(
|
||||
redisService.deleteStreamEntries.mock.invocationCallOrder[0],
|
||||
redisService.acknowledgeAndDeleteStream.mock.invocationCallOrder[0],
|
||||
);
|
||||
});
|
||||
|
||||
@@ -249,7 +351,6 @@ describe('RiotJobWorkerService', () => {
|
||||
message: { job: JSON.stringify(job) },
|
||||
});
|
||||
|
||||
expect(redisService.acknowledgeStream).not.toHaveBeenCalled();
|
||||
expect(redisService.deleteStreamEntries).not.toHaveBeenCalled();
|
||||
expect(redisService.acknowledgeAndDeleteStream).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user