Skip to content

fix: upstream database and api performance improvements #137

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Cactus-15-49 merged 4 commits into from
Apr 24, 2023
Merged

fix: upstream database and api performance improvements #137

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
985977a
feat(database): use second database connection for api
sebastijankuzner Mar 10, 2023
409d956
fix: make pagination service non-blocking
sebastijankuzner Mar 10, 2023
f6ac71f
feat(api): query semaphore plugin
sebastijankuzner Mar 16, 2023
cb8ca0a
fix(api): skip logging query failed errors
sebastijankuzner Mar 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions packages/api/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
"node-cache": "5.1.2",
"qs": "6.10.3",
"rate-limiter-flexible": "1.3.2",
"semaphore": "1.1.0",
"semver": "6.3.0"
},
"devDependencies": {
Expand All @@ -42,6 +43,7 @@
"@types/hapi__nes": "11.0.5",
"@types/ip": "1.1.0",
"@types/qs": "6.9.7",
"@types/semaphore": "1.1.1",
"@types/semver": "6.2.3",
"lodash.clonedeep": "4.5.0"
}
Expand Down
3 changes: 3 additions & 0 deletions packages/api/src/controllers/blockchain.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,18 @@ import { Controller } from "./controller";

export class BlockchainController extends Controller {
@Container.inject(Container.Identifiers.BlockHistoryService)
@Container.tagged("connection", "api")
private readonly blockHistoryService!: Contracts.Shared.BlockHistoryService;

@Container.inject(Container.Identifiers.StateStore)
private readonly stateStore!: Contracts.State.StateStore;

@Container.inject(Container.Identifiers.TransactionHistoryService)
@Container.tagged("connection", "api")
private readonly transactionHistoryService!: Contracts.Shared.TransactionHistoryService;

@Container.inject(Container.Identifiers.DatabaseTransactionRepository)
@Container.tagged("connection", "api")
private readonly transactionRepository!: Repositories.TransactionRepository;

@Container.inject(Container.Identifiers.WalletRepository)
Expand Down
3 changes: 3 additions & 0 deletions packages/api/src/controllers/blocks.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,15 @@ export class BlocksController extends Controller {
private readonly blockchain!: Contracts.Blockchain.Blockchain;

@Container.inject(Container.Identifiers.BlockHistoryService)
@Container.tagged("connection", "api")
private readonly blockHistoryService!: Contracts.Shared.BlockHistoryService;

@Container.inject(Container.Identifiers.MissedBlockHistoryService)
@Container.tagged("connection", "api")
private readonly missedBlockHistoryService!: Contracts.Shared.MissedBlockHistoryService;

@Container.inject(Container.Identifiers.TransactionHistoryService)
@Container.tagged("connection", "api")
private readonly transactionHistoryService!: Contracts.Shared.TransactionHistoryService;

public async index(request: Hapi.Request, h: Hapi.ResponseToolkit): Promise<object> {
Expand Down
12 changes: 10 additions & 2 deletions packages/api/src/controllers/delegates.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,17 @@ export class DelegatesController extends Controller {
private readonly walletSearchService!: WalletSearchService;

@Container.inject(Container.Identifiers.BlockHistoryService)
@Container.tagged("connection", "api")
private readonly blockHistoryService!: Contracts.Shared.BlockHistoryService;

@Container.inject(Container.Identifiers.MissedBlockHistoryService)
@Container.tagged("connection", "api")
private readonly missedBlockHistoryService!: Contracts.Shared.MissedBlockHistoryService;

public index(request: Hapi.Request, h: Hapi.ResponseToolkit): Contracts.Search.ResultsPage<DelegateResource> {
public async index(
request: Hapi.Request,
h: Hapi.ResponseToolkit,
): Promise<Contracts.Search.ResultsPage<DelegateResource>> {
const pagination = this.getQueryPagination(request.query);
const sorting = request.query.orderBy as Contracts.Search.Sorting;
const criteria = this.getQueryCriteria(request.query, delegateCriteriaSchemaObject) as DelegateCriteria;
Expand All @@ -54,7 +59,10 @@ export class DelegatesController extends Controller {
return { data: delegateResource };
}

public voters(request: Hapi.Request, h: Hapi.ResponseToolkit): Contracts.Search.ResultsPage<WalletResource> | Boom {
public async voters(
request: Hapi.Request,
h: Hapi.ResponseToolkit,
): Promise<Contracts.Search.ResultsPage<WalletResource> | Boom> {
const walletId = request.params.id as string;

const walletResource = this.walletSearchService.getWallet(walletId);
Expand Down
6 changes: 5 additions & 1 deletion packages/api/src/controllers/locks.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,13 @@ export class LocksController extends Controller {
private readonly lockSearchService!: LockSearchService;

@Container.inject(Container.Identifiers.TransactionHistoryService)
@Container.tagged("connection", "api")
private readonly transactionHistoryService!: Contracts.Shared.TransactionHistoryService;

public index(request: Hapi.Request, h: Hapi.ResponseToolkit): Contracts.Search.ResultsPage<LockResource> {
public async index(
request: Hapi.Request,
h: Hapi.ResponseToolkit,
): Promise<Contracts.Search.ResultsPage<LockResource>> {
const pagination = this.getQueryPagination(request.query);
const sorting = request.query.orderBy as Contracts.Search.Sorting;
const criteria = this.getQueryCriteria(request.query, lockCriteriaSchemaObject) as LockCriteria;
Expand Down
1 change: 1 addition & 0 deletions packages/api/src/controllers/node.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ export class NodeController extends Controller {
private readonly networkMonitor!: Contracts.P2P.NetworkMonitor;

@Container.inject(Container.Identifiers.DatabaseTransactionRepository)
@Container.tagged("connection", "api")
private readonly transactionRepository!: Repositories.TransactionRepository;

public async status(
Expand Down
1 change: 1 addition & 0 deletions packages/api/src/controllers/rounds.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { Controller } from "./controller";

export class RoundsController extends Controller {
@Container.inject(Container.Identifiers.DatabaseRoundRepository)
@Container.tagged("connection", "api")
private readonly roundRepository!: Repositories.RoundRepository;

public async delegates(request: Hapi.Request, h: Hapi.ResponseToolkit): Promise<object | Boom.Boom> {
Expand Down
2 changes: 2 additions & 0 deletions packages/api/src/controllers/transactions.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,11 @@ export class TransactionsController extends Controller {
private readonly poolQuery!: Contracts.Pool.Query;

@Container.inject(Container.Identifiers.TransactionHistoryService)
@Container.tagged("connection", "api")
private readonly transactionHistoryService!: Contracts.Shared.TransactionHistoryService;

@Container.inject(Container.Identifiers.BlockHistoryService)
@Container.tagged("connection", "api")
private readonly blockHistoryService!: Contracts.Shared.BlockHistoryService;

@Container.inject(Container.Identifiers.PoolProcessor)
Expand Down
1 change: 1 addition & 0 deletions packages/api/src/controllers/votes.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { Controller } from "./controller";
@Container.injectable()
export class VotesController extends Controller {
@Container.inject(Container.Identifiers.TransactionHistoryService)
@Container.tagged("connection", "api")
private readonly transactionHistoryService!: Contracts.Shared.TransactionHistoryService;

public async index(request: Hapi.Request, h: Hapi.ResponseToolkit): Promise<Contracts.Search.ResultsPage<object>> {
Expand Down
16 changes: 13 additions & 3 deletions packages/api/src/controllers/wallets.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,17 +28,24 @@ export class WalletsController extends Controller {
private readonly lockSearchService!: LockSearchService;

@Container.inject(Container.Identifiers.TransactionHistoryService)
@Container.tagged("connection", "api")
private readonly transactionHistoryService!: Contracts.Shared.TransactionHistoryService;

public index(request: Hapi.Request, h: Hapi.ResponseToolkit): Contracts.Search.ResultsPage<WalletResource> {
public async index(
request: Hapi.Request,
h: Hapi.ResponseToolkit,
): Promise<Contracts.Search.ResultsPage<WalletResource>> {
const pagination = this.getQueryPagination(request.query);
const sorting = request.query.orderBy as Contracts.Search.Sorting;
const criteria = this.getQueryCriteria(request.query, walletCriteriaSchemaObject) as WalletCriteria;

return this.walletSearchService.getWalletsPage(pagination, sorting, criteria);
}

public top(request: Hapi.Request, h: Hapi.ResponseToolkit): Contracts.Search.ResultsPage<WalletResource> {
public async top(
request: Hapi.Request,
h: Hapi.ResponseToolkit,
): Promise<Contracts.Search.ResultsPage<WalletResource>> {
const pagination = this.getQueryPagination(request.query);
const sorting = request.query.orderBy as Contracts.Search.Sorting;
const criteria = this.getQueryCriteria(request.query, walletCriteriaSchemaObject) as WalletCriteria;
Expand All @@ -57,7 +64,10 @@ export class WalletsController extends Controller {
return { data: walletResource };
}

public locks(request: Hapi.Request, h: Hapi.ResponseToolkit): Contracts.Search.ResultsPage<LockResource> | Boom {
public async locks(
request: Hapi.Request,
h: Hapi.ResponseToolkit,
): Promise<Contracts.Search.ResultsPage<LockResource> | Boom> {
const walletId = request.params.id as string;
const walletResource = this.walletSearchService.getWallet(walletId);

Expand Down
25 changes: 25 additions & 0 deletions packages/api/src/defaults.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,31 @@ export const defaults = {
stdTTL: 8,
checkperiod: 120,
},
semaphore: {
enabled: !process.env.CORE_API_SEMAPHORE_DISABLED,
database: {
levelOne: {
concurrency: 10,
queueLimit: 100,
maxOffset: 10000,
},
levelTwo: {
concurrency: 1,
queueLimit: 5,
},
},
memory: {
levelOne: {
concurrency: 3,
queueLimit: 30,
maxOffset: 1000,
},
levelTwo: {
concurrency: 1,
queueLimit: 5,
},
},
},
rateLimit: {
enabled: !process.env.CORE_API_RATE_LIMIT_DISABLED,
points: process.env.CORE_API_RATE_LIMIT_USER_LIMIT || 100,
Expand Down
2 changes: 2 additions & 0 deletions packages/api/src/plugins/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { log } from "./log";
import { pagination } from "./pagination";
import { rateLimit } from "./rate-limit";
import { responseHeaders } from "./response-headers";
import { semaphore } from "./semaphore";
import { whitelist } from "./whitelist";

const isListed = (ip: string, patterns: string[]): boolean => {
Expand Down Expand Up @@ -79,6 +80,7 @@ export const preparePlugins = (config) => {
},
{ plugin: commaArrayQuery },
{ plugin: dotSeparatedQuery },
{ plugin: semaphore, options: config.semaphore },
{
plugin: cache,
options: config.plugins.cache,
Expand Down
Loading