fix: recover stale auction durability history
This commit is contained in:
1 parent
37f1f4225d
commit
8567948084
2 files changed
+294
No files matched your search
@@ -446,6 +446,216 @@ describe('auction durability projection helpers', () => {
|
|||||||
expect(auctionUpdateOne).not.toHaveBeenCalled();
|
expect(auctionUpdateOne).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('recovers stale append payloads as history-only records', async () => {
|
||||||
|
const session = {
|
||||||
|
abortTransaction: jest.fn().mockResolvedValue(undefined),
|
||||||
|
commitTransaction: jest.fn().mockResolvedValue(undefined),
|
||||||
|
endSession: jest.fn().mockResolvedValue(undefined),
|
||||||
|
startTransaction: jest.fn(),
|
||||||
|
};
|
||||||
|
const historyFindOne = jest.fn().mockReturnValue({
|
||||||
|
session: jest.fn().mockReturnValue({
|
||||||
|
lean: jest.fn().mockReturnValue({
|
||||||
|
exec: jest.fn().mockResolvedValue(null),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
const historyUpdateOne = jest.fn().mockReturnValue({
|
||||||
|
exec: jest.fn().mockResolvedValue({ upsertedCount: 1 }),
|
||||||
|
});
|
||||||
|
const auctionUpdateOne = jest.fn();
|
||||||
|
const sideEffectUpdateOne = jest.fn();
|
||||||
|
const projectionExec = jest.fn().mockResolvedValue({
|
||||||
|
auctionLifecycleId: 'life-1',
|
||||||
|
revision: 10,
|
||||||
|
});
|
||||||
|
const auctionFindById = jest.fn().mockReturnValue({
|
||||||
|
select: jest.fn().mockReturnValue({
|
||||||
|
session: jest.fn().mockReturnValue({
|
||||||
|
lean: jest.fn().mockReturnValue({ exec: projectionExec }),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
const scheduler = new AuctionDurabilityScheduler(
|
||||||
|
{
|
||||||
|
createSession: jest.fn().mockResolvedValue(session),
|
||||||
|
getModel: jest.fn((name: string) => {
|
||||||
|
if (name === 'auctionHistory') {
|
||||||
|
return {
|
||||||
|
findOne: historyFindOne,
|
||||||
|
updateOne: historyUpdateOne,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (name === 'auction') {
|
||||||
|
return { findById: auctionFindById, updateOne: auctionUpdateOne };
|
||||||
|
}
|
||||||
|
return { updateOne: sideEffectUpdateOne };
|
||||||
|
}),
|
||||||
|
} as never,
|
||||||
|
{ raw: jest.fn() } as never,
|
||||||
|
);
|
||||||
|
|
||||||
|
await scheduler.applyDurabilityPayload({
|
||||||
|
...payload,
|
||||||
|
matchSideEffectMutation: { type: 'syncMatchTeamsAfterComplete' },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(historyFindOne).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
auctionId: 'auction-1',
|
||||||
|
sequence: { $gt: 4 },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(historyUpdateOne).toHaveBeenCalledWith(
|
||||||
|
{
|
||||||
|
auctionId: 'auction-1',
|
||||||
|
auctionStateTransitionId: 'transition-1',
|
||||||
|
},
|
||||||
|
expect.objectContaining({
|
||||||
|
$setOnInsert: expect.objectContaining({
|
||||||
|
sequence: 4,
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
{ session, upsert: true },
|
||||||
|
);
|
||||||
|
expect(auctionUpdateOne).not.toHaveBeenCalled();
|
||||||
|
expect(sideEffectUpdateOne).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not resurrect stale append payloads invalidated by later undo', async () => {
|
||||||
|
const session = {
|
||||||
|
abortTransaction: jest.fn().mockResolvedValue(undefined),
|
||||||
|
commitTransaction: jest.fn().mockResolvedValue(undefined),
|
||||||
|
endSession: jest.fn().mockResolvedValue(undefined),
|
||||||
|
startTransaction: jest.fn(),
|
||||||
|
};
|
||||||
|
const historyFindOne = jest.fn().mockReturnValue({
|
||||||
|
session: jest.fn().mockReturnValue({
|
||||||
|
lean: jest.fn().mockReturnValue({
|
||||||
|
exec: jest.fn().mockResolvedValue({
|
||||||
|
historyMutation: {
|
||||||
|
targetHistorySequence: 2,
|
||||||
|
type: 'deleteAfterSequenceAndAppendUndo',
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
const historyUpdateOne = jest.fn();
|
||||||
|
const auctionUpdateOne = jest.fn();
|
||||||
|
const projectionExec = jest.fn().mockResolvedValue({
|
||||||
|
auctionLifecycleId: 'life-1',
|
||||||
|
revision: 10,
|
||||||
|
});
|
||||||
|
const auctionFindById = jest.fn().mockReturnValue({
|
||||||
|
select: jest.fn().mockReturnValue({
|
||||||
|
session: jest.fn().mockReturnValue({
|
||||||
|
lean: jest.fn().mockReturnValue({ exec: projectionExec }),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
const scheduler = new AuctionDurabilityScheduler(
|
||||||
|
{
|
||||||
|
createSession: jest.fn().mockResolvedValue(session),
|
||||||
|
getModel: jest.fn((name: string) => {
|
||||||
|
if (name === 'auctionHistory') {
|
||||||
|
return {
|
||||||
|
findOne: historyFindOne,
|
||||||
|
updateOne: historyUpdateOne,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (name === 'auction') {
|
||||||
|
return { findById: auctionFindById, updateOne: auctionUpdateOne };
|
||||||
|
}
|
||||||
|
return { updateOne: jest.fn() };
|
||||||
|
}),
|
||||||
|
} as never,
|
||||||
|
{ raw: jest.fn() } as never,
|
||||||
|
);
|
||||||
|
|
||||||
|
await scheduler.applyDurabilityPayload(payload);
|
||||||
|
|
||||||
|
expect(historyFindOne).toHaveBeenCalled();
|
||||||
|
expect(historyUpdateOne).not.toHaveBeenCalled();
|
||||||
|
expect(auctionUpdateOne).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('clears stale append payloads after history-only recovery', async () => {
|
||||||
|
const sendCommand = jest
|
||||||
|
.fn()
|
||||||
|
.mockResolvedValueOnce(['payload-1'])
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
committedAtRedisTimestampMS: '1780000000000',
|
||||||
|
payloadJson: JSON.stringify({
|
||||||
|
...payload,
|
||||||
|
matchSideEffectMutation: { type: 'syncMatchTeamsAfterComplete' },
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
.mockResolvedValueOnce(1)
|
||||||
|
.mockResolvedValueOnce(1);
|
||||||
|
const historyFindOne = jest.fn().mockReturnValue({
|
||||||
|
session: jest.fn().mockReturnValue({
|
||||||
|
lean: jest.fn().mockReturnValue({
|
||||||
|
exec: jest.fn().mockResolvedValue(null),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
const historyUpdateOne = jest.fn().mockReturnValue({
|
||||||
|
exec: jest.fn().mockResolvedValue({ upsertedCount: 1 }),
|
||||||
|
});
|
||||||
|
const auctionUpdateOne = jest.fn();
|
||||||
|
const sideEffectUpdateOne = jest.fn();
|
||||||
|
const auctionFindById = jest.fn().mockReturnValue({
|
||||||
|
select: jest.fn().mockReturnValue({
|
||||||
|
session: jest.fn().mockReturnValue({
|
||||||
|
lean: jest.fn().mockReturnValue({
|
||||||
|
exec: jest.fn().mockResolvedValue({
|
||||||
|
auctionLifecycleId: 'life-1',
|
||||||
|
revision: 10,
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
const scheduler = new AuctionDurabilityScheduler(
|
||||||
|
{
|
||||||
|
createSession: jest.fn().mockResolvedValue(null),
|
||||||
|
getModel: jest.fn((name: string) => {
|
||||||
|
if (name === 'auctionHistory') {
|
||||||
|
return {
|
||||||
|
findOne: historyFindOne,
|
||||||
|
updateOne: historyUpdateOne,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (name === 'auction') {
|
||||||
|
return { findById: auctionFindById, updateOne: auctionUpdateOne };
|
||||||
|
}
|
||||||
|
return { updateOne: sideEffectUpdateOne };
|
||||||
|
}),
|
||||||
|
} as never,
|
||||||
|
{
|
||||||
|
getCurrentRedisTimestampMS: jest.fn().mockResolvedValue(1000),
|
||||||
|
raw: jest.fn().mockReturnValue({ sendCommand }),
|
||||||
|
} as never,
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(scheduler.processDueDurabilityPayloads(10)).resolves.toEqual({
|
||||||
|
applied: 1,
|
||||||
|
failed: 0,
|
||||||
|
skipped: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(historyUpdateOne).toHaveBeenCalledTimes(1);
|
||||||
|
expect(auctionUpdateOne).not.toHaveBeenCalled();
|
||||||
|
expect(sideEffectUpdateOne).not.toHaveBeenCalled();
|
||||||
|
expect(sendCommand).toHaveBeenNthCalledWith(3, [
|
||||||
|
'ZREM',
|
||||||
|
'auction:auctionDurabilityRetryQueue',
|
||||||
|
'payload-1',
|
||||||
|
]);
|
||||||
|
expect(sendCommand).toHaveBeenNthCalledWith(4, ['DEL', 'payload-1']);
|
||||||
|
});
|
||||||
|
|
||||||
it('processes due payloads and clears successful outbox entries', async () => {
|
it('processes due payloads and clears successful outbox entries', async () => {
|
||||||
const sendCommand = jest
|
const sendCommand = jest
|
||||||
.fn()
|
.fn()
|
||||||
|
|||||||
@@ -242,6 +242,9 @@ export class AuctionDurabilityScheduler
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
session?.startTransaction();
|
session?.startTransaction();
|
||||||
|
const historyMutation = parseAuctionHistoryMutation(
|
||||||
|
payload.historyMutation,
|
||||||
|
);
|
||||||
if (typeof AuctionModel.findById === 'function') {
|
if (typeof AuctionModel.findById === 'function') {
|
||||||
const currentProjection =
|
const currentProjection =
|
||||||
await readAuctionProjectionIdentityForMutation(
|
await readAuctionProjectionIdentityForMutation(
|
||||||
@@ -252,6 +255,18 @@ export class AuctionDurabilityScheduler
|
|||||||
if (
|
if (
|
||||||
!shouldProcessAuctionDurabilityPayload(currentProjection, payload)
|
!shouldProcessAuctionDurabilityPayload(currentProjection, payload)
|
||||||
) {
|
) {
|
||||||
|
// 뒤 payload가 먼저 projection revision을 올리면 앞 append payload가 stale이 된다.
|
||||||
|
// 이때 projection/side effect는 건드리지 않고 undo용 history만 멱등 복구한다.
|
||||||
|
if (
|
||||||
|
historyMutation.type === 'append' &&
|
||||||
|
currentProjection.auctionLifecycleId === payload.auctionLifecycleId
|
||||||
|
) {
|
||||||
|
await this.applyStaleAppendHistoryRecovery({
|
||||||
|
AuctionHistoryModel,
|
||||||
|
payload,
|
||||||
|
session,
|
||||||
|
});
|
||||||
|
}
|
||||||
if (session) await session.commitTransaction();
|
if (session) await session.commitTransaction();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -516,6 +531,75 @@ export class AuctionDurabilityScheduler
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async applyStaleAppendHistoryRecovery(params: {
|
||||||
|
AuctionHistoryModel: {
|
||||||
|
findOne?: (...args: never[]) => unknown;
|
||||||
|
updateOne: (...args: never[]) => unknown;
|
||||||
|
};
|
||||||
|
payload: AuctionDurabilityPayload;
|
||||||
|
session: mongoose.ClientSession | null | undefined;
|
||||||
|
}): Promise<void> {
|
||||||
|
const invalidated =
|
||||||
|
await this.isAppendHistoryInvalidatedByLaterDestructiveMutation(params);
|
||||||
|
if (invalidated) return;
|
||||||
|
|
||||||
|
const historyOperation = buildAuctionHistoryUpsert(params.payload).updateOne;
|
||||||
|
await execMaybe(
|
||||||
|
params.AuctionHistoryModel.updateOne(
|
||||||
|
historyOperation.filter as never,
|
||||||
|
historyOperation.update as never,
|
||||||
|
{
|
||||||
|
session: params.session,
|
||||||
|
upsert: true,
|
||||||
|
} as never,
|
||||||
|
) as Promise<unknown> | { exec(): Promise<unknown> },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async isAppendHistoryInvalidatedByLaterDestructiveMutation(params: {
|
||||||
|
AuctionHistoryModel: {
|
||||||
|
findOne?: (...args: never[]) => unknown;
|
||||||
|
};
|
||||||
|
payload: AuctionDurabilityPayload;
|
||||||
|
session: mongoose.ClientSession | null | undefined;
|
||||||
|
}): Promise<boolean> {
|
||||||
|
if (!params.AuctionHistoryModel.findOne) return true;
|
||||||
|
|
||||||
|
// 늦게 도착한 append는 history gap을 메울 수 있지만,
|
||||||
|
// 이후 undo/reset이 의도적으로 삭제한 sequence를 되살리면 안 된다.
|
||||||
|
const query = params.AuctionHistoryModel.findOne({
|
||||||
|
auctionId: params.payload.auctionId,
|
||||||
|
sequence: { $gt: params.payload.auctionHistorySequence },
|
||||||
|
$or: [
|
||||||
|
{
|
||||||
|
'historyMutation.targetHistorySequence': {
|
||||||
|
$lt: params.payload.auctionHistorySequence,
|
||||||
|
},
|
||||||
|
'historyMutation.type': 'deleteAfterSequenceAndAppendUndo',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'historyMutation.resetMode': { $ne: 'preserve-audit' },
|
||||||
|
'historyMutation.type': 'reset',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
} as never);
|
||||||
|
const scoped =
|
||||||
|
query && typeof query === 'object' && 'session' in query
|
||||||
|
? (query as { session: (...args: never[]) => unknown }).session(
|
||||||
|
params.session as never,
|
||||||
|
)
|
||||||
|
: query;
|
||||||
|
const leaned =
|
||||||
|
scoped && typeof scoped === 'object' && 'lean' in scoped
|
||||||
|
? (scoped as { lean: () => unknown }).lean()
|
||||||
|
: scoped;
|
||||||
|
const result =
|
||||||
|
leaned && typeof leaned === 'object' && 'exec' in leaned
|
||||||
|
? await (leaned as { exec: () => Promise<unknown> }).exec()
|
||||||
|
: await Promise.resolve(leaned);
|
||||||
|
return Boolean(result);
|
||||||
|
}
|
||||||
|
|
||||||
private async applyResetMatchSideEffect(
|
private async applyResetMatchSideEffect(
|
||||||
payload: AuctionDurabilityPayload,
|
payload: AuctionDurabilityPayload,
|
||||||
mutation: ParsedMatchSideEffectMutation,
|
mutation: ParsedMatchSideEffectMutation,
|
||||||
|
|||||||
Reference in new issue
Block a user