feat: redesign auction engine with redis sot
This commit is contained in:
1 parent
737702d78c
commit
e8bb20d988
75 files changed
+12737
-542
No files matched your search
@@ -0,0 +1,205 @@
|
||||
import { AuctionRedisService } from './redis/auction-redis.service';
|
||||
import { AuctionDeadlineScheduler } from './auction-deadline.scheduler';
|
||||
import { AuctionRedisCommandService } from './redis/auction-redis-command.service';
|
||||
import { AuctionService } from './auction.service';
|
||||
|
||||
function makeScheduler(params: {
|
||||
dueReply?: unknown;
|
||||
leaderLockAcquired?: boolean;
|
||||
expireReply?: unknown;
|
||||
completedState?: unknown;
|
||||
currentRedisTimestampMS?: number;
|
||||
}) {
|
||||
const sendCommand = jest.fn().mockResolvedValue(params.dueReply ?? []);
|
||||
const redis = {
|
||||
acquireTokenLock: jest
|
||||
.fn()
|
||||
.mockResolvedValue(params.leaderLockAcquired ?? true),
|
||||
getCurrentRedisTimestampMS: jest
|
||||
.fn()
|
||||
.mockResolvedValue(params.currentRedisTimestampMS ?? 1000),
|
||||
releaseTokenLock: jest.fn().mockResolvedValue(true),
|
||||
raw: jest.fn().mockReturnValue({ sendCommand }),
|
||||
} as unknown as jest.Mocked<AuctionRedisService>;
|
||||
const redisCommand = {
|
||||
expirePrepare: jest.fn().mockResolvedValue(
|
||||
params.expireReply ?? {
|
||||
auctionStateTransitionId: 'expire-1',
|
||||
bidRoundExpiryDecision: 'passed',
|
||||
currentServerTimestampMS: 1000,
|
||||
ok: true,
|
||||
revision: 2,
|
||||
},
|
||||
),
|
||||
} as unknown as jest.Mocked<AuctionRedisCommandService>;
|
||||
const auctionService = {
|
||||
completePreparedExpiry: jest
|
||||
.fn()
|
||||
.mockResolvedValue(params.completedState ?? { phase: 'passed' }),
|
||||
} as unknown as jest.Mocked<AuctionService>;
|
||||
return {
|
||||
auctionService,
|
||||
redis,
|
||||
redisCommand,
|
||||
scheduler: new AuctionDeadlineScheduler(
|
||||
redis,
|
||||
redisCommand,
|
||||
auctionService,
|
||||
),
|
||||
sendCommand,
|
||||
};
|
||||
}
|
||||
|
||||
describe('AuctionDeadlineScheduler', () => {
|
||||
afterEach(() => {
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('starts and clears the deadline tick timer', () => {
|
||||
const setIntervalSpy = jest.spyOn(global, 'setInterval');
|
||||
const clearIntervalSpy = jest.spyOn(global, 'clearInterval');
|
||||
const { scheduler } = makeScheduler({});
|
||||
|
||||
scheduler.onModuleInit();
|
||||
scheduler.onModuleDestroy();
|
||||
|
||||
expect(setIntervalSpy).toHaveBeenCalledWith(expect.any(Function), 1000);
|
||||
expect(clearIntervalSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('processes a deadline tick only while holding the Redis leader lock', async () => {
|
||||
const { redis, scheduler } = makeScheduler({});
|
||||
const processDueBidRoundDeadlinesSpy = jest
|
||||
.spyOn(scheduler, 'processDueBidRoundDeadlines')
|
||||
.mockResolvedValue({ completed: 0, prepared: 0, skipped: 0 });
|
||||
|
||||
await (scheduler as unknown as { safeTick: () => Promise<void> }).safeTick();
|
||||
|
||||
const leaderLockToken = redis.acquireTokenLock.mock.calls[0]?.[1];
|
||||
expect(redis.acquireTokenLock).toHaveBeenCalledWith(
|
||||
'auction:bidRoundDeadlineScheduler:leader',
|
||||
expect.any(String),
|
||||
2500,
|
||||
);
|
||||
expect(processDueBidRoundDeadlinesSpy).toHaveBeenCalledWith(20);
|
||||
expect(redis.releaseTokenLock).toHaveBeenCalledWith(
|
||||
'auction:bidRoundDeadlineScheduler:leader',
|
||||
leaderLockToken,
|
||||
);
|
||||
});
|
||||
|
||||
it('skips deadline tick work when another instance holds the leader lock', async () => {
|
||||
const { redis, scheduler } = makeScheduler({ leaderLockAcquired: false });
|
||||
const processDueBidRoundDeadlinesSpy = jest
|
||||
.spyOn(scheduler, 'processDueBidRoundDeadlines')
|
||||
.mockResolvedValue({ completed: 0, prepared: 0, skipped: 0 });
|
||||
|
||||
await (scheduler as unknown as { safeTick: () => Promise<void> }).safeTick();
|
||||
|
||||
expect(processDueBidRoundDeadlinesSpy).not.toHaveBeenCalled();
|
||||
expect(redis.releaseTokenLock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('reads due match ids from the Redis deadline queue with a batch limit', async () => {
|
||||
const { redis, scheduler } = makeScheduler({
|
||||
dueReply: ['match-1', 'match-2'],
|
||||
});
|
||||
|
||||
await expect(scheduler.readDueMatchIds(1_000, 2)).resolves.toEqual([
|
||||
'match-1',
|
||||
'match-2',
|
||||
]);
|
||||
expect(redis.raw().sendCommand).toHaveBeenCalledWith([
|
||||
'ZRANGEBYSCORE',
|
||||
'auction:bidRoundDeadlineQueue',
|
||||
'-inf',
|
||||
'1000',
|
||||
'LIMIT',
|
||||
'0',
|
||||
'2',
|
||||
]);
|
||||
});
|
||||
|
||||
it('prepares and completes due auction deadlines', async () => {
|
||||
const matchId = '665f2f01f1f1f1f1f1f1f1f1';
|
||||
const { auctionService, redisCommand, scheduler } = makeScheduler({
|
||||
dueReply: [matchId],
|
||||
});
|
||||
|
||||
await expect(scheduler.processDueBidRoundDeadlines(10)).resolves.toEqual({
|
||||
completed: 1,
|
||||
prepared: 1,
|
||||
skipped: 0,
|
||||
});
|
||||
expect(redisCommand.expirePrepare).toHaveBeenCalledWith(
|
||||
matchId,
|
||||
expect.objectContaining({
|
||||
bidRoundExpiryRetryDelayMS: 3000,
|
||||
matchId,
|
||||
}),
|
||||
);
|
||||
expect(auctionService.completePreparedExpiry).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('skips noop expiry preparations', async () => {
|
||||
const { auctionService, scheduler } = makeScheduler({
|
||||
dueReply: ['665f2f01f1f1f1f1f1f1f1f1'],
|
||||
expireReply: {
|
||||
code: 'NOT_DUE',
|
||||
currentServerTimestampMS: 1000,
|
||||
noop: true,
|
||||
},
|
||||
});
|
||||
|
||||
await expect(scheduler.processDueBidRoundDeadlines(10)).resolves.toEqual({
|
||||
completed: 0,
|
||||
prepared: 0,
|
||||
skipped: 1,
|
||||
});
|
||||
expect(auctionService.completePreparedExpiry).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('removes invalid deadline queue members', async () => {
|
||||
const { scheduler, sendCommand } = makeScheduler({
|
||||
dueReply: ['not-a-match-id'],
|
||||
});
|
||||
|
||||
await expect(scheduler.processDueBidRoundDeadlines(10)).resolves.toEqual({
|
||||
completed: 0,
|
||||
prepared: 0,
|
||||
skipped: 1,
|
||||
});
|
||||
expect(sendCommand).toHaveBeenLastCalledWith([
|
||||
'ZREM',
|
||||
'auction:bidRoundDeadlineQueue',
|
||||
'not-a-match-id',
|
||||
]);
|
||||
});
|
||||
|
||||
it('continues processing later deadline items after one item fails', async () => {
|
||||
const matchIds = [
|
||||
'665f2f01f1f1f1f1f1f1f1f1',
|
||||
'665f2f02f1f1f1f1f1f1f1f1',
|
||||
];
|
||||
const { auctionService, redisCommand, scheduler } = makeScheduler({
|
||||
dueReply: matchIds,
|
||||
});
|
||||
redisCommand.expirePrepare
|
||||
.mockRejectedValueOnce(new Error('temporary redis error'))
|
||||
.mockResolvedValueOnce({
|
||||
auctionStateTransitionId: 'expire-2',
|
||||
bidRoundExpiryDecision: 'passed',
|
||||
currentServerTimestampMS: 1000,
|
||||
ok: true,
|
||||
revision: 3,
|
||||
});
|
||||
|
||||
await expect(scheduler.processDueBidRoundDeadlines(10)).resolves.toEqual({
|
||||
completed: 1,
|
||||
prepared: 1,
|
||||
skipped: 1,
|
||||
});
|
||||
expect(redisCommand.expirePrepare).toHaveBeenCalledTimes(2);
|
||||
expect(auctionService.completePreparedExpiry).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user