diff --git a/config.example.yaml b/config.example.yaml index e69de29..737f96d 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -0,0 +1,46 @@ +labels_allow: + __name__: + - up + - messenger_dialog_info + - messenger_dialog_messages_count + - messenger_dialog_keywords_count + job: + - mtproto_exporter + instance: + - some_prometheus_instance:9669 + peerId: + - "-9001234567890" + - "-9000987654321" + - "-9001122334455" + - "-9006677889900" + peerType: true + +queries_allow: + - 1+1 + - sum(increase(messenger_dialog_keywords_count{job="$job", peerId=~"$allowed_peers", peerId=~"$chat"}[30d])) by (keyword) + - >- + increase(messenger_dialog_messages_count{job="$job", peerId=~"$allowed_peers", peerId=~"$chat"}[30d]) * + on (peerId) group_right messenger_dialog_info{job="$job", peerId=~"$allowed_peers", peerId=~"$chat"} + - >- + sum(rate(messenger_dialog_keywords_count{job="$job", peerId=~"$allowed_peers", peerId=~"$chat"}[1h]) + or (avg_over_time(messenger_dialog_keywords_count{job="$job", peerId=~"$allowed_peers", peerId=~"$chat"}[$__range]) * 0)) + by (keyword) * 3600 + +variables: + job: mtproto_exporter + allowed_peers: + separator: "|" + all_of: + - "-9001234567890" + - "-9000987654321" + - "-9001122334455" + - "-9006677889900" + chat: + separator: "|" + any_of: + - "-9001234567890" + - "-9000987654321" + - "-9001122334455" + - "-9006677889900" + __range: + pattern: "^[0-9]+[ywdhms]?$" diff --git a/src/config.ts b/src/config.ts index da4d77c..6b8cf1f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -3,6 +3,7 @@ import { readFile } from "node:fs/promises"; import fs from "node:fs"; import cmdline from "command-line-args"; import yaml from "js-yaml"; +import { compileQuery, Query } from "./query.js"; export interface Configuration { bindHost: string; @@ -10,9 +11,49 @@ export interface Configuration { configFile: string; watchFile: boolean; prometheusURL: string; - permit_query: string[]; + queriesAllow: string[]; + variables: AnyVariable[]; + queriesCompiled: Query[]; + labelsAllow: LabelAllow[]; } +interface LabelAllow { + name: string; + values?: string[]; +} + +interface BaseVariable { + name: string; + varType: T +} + +export interface LiteralVariable extends BaseVariable<"literal"> { + text: string; +} + +export function isAnyVariable(o: any): o is AnyVariable { + return ( + typeof o === "object" && + typeof o.name === "string" && + typeof o.varType === "string" + ); +} + +interface ArrayLikeVariable extends BaseVariable<`${T}_array`> { + items: string[]; + separator: string; +} + +export type AnyOfItemsVariable = ArrayLikeVariable<"any_of">; +export type AllOfItemsVariable = ArrayLikeVariable<"all_of">; + +export interface PatternVariable extends BaseVariable<"pattern"> { + pattern: RegExp; +} + +export type AnyVariable = LiteralVariable | AnyOfItemsVariable | AllOfItemsVariable | PatternVariable; + + const optionDefinitions: OptionDefinition[] = [ { name: "bind-host", alias: "b", type: String, defaultValue: "0.0.0.0" }, { name: "port", alias: "p", type: Number, defaultValue: 9091 }, @@ -33,22 +74,129 @@ const config: Configuration = { configFile: cli["config-file"], watchFile: cli["watch-file"], prometheusURL: cli["prometheus-url"], - permit_query: [], + queriesAllow: [], + variables: [], + queriesCompiled: [], + labelsAllow: [], }; -export async function loadConfigFile(filePath: string): Promise { - const doc = yaml.load(await readFile(filePath, "utf8")) as { permit_query?: any[] }; +function arrayProp(propName: string, target: any): string[] | undefined { + if (!(propName in target)) { + return; + } + if (!Array.isArray(target[propName])) { + throw new Error(`property '${propName}' should be an array.`); + } + const badElementIndex = target[propName].findIndex(v => typeof v !== "string" && typeof v !== "number"); + if (badElementIndex !== -1) { + throw new Error(`all elements of '${propName}' should be strings or numbers. offending element index is ${badElementIndex}.`); + } + return target[propName]; +} - if (typeof doc.permit_query === "object" && doc.permit_query.constructor.name === "Array") { - for (const item of doc.permit_query) { - if (typeof item === "string") { - config.permit_query.push(item); - // } else if (typeof item === "object") { +function loadConfigFileVariables(data: object): AnyVariable[] { + const vars: AnyVariable[] = []; + for (const [name, value] of Object.entries(data)) { + if (typeof value === "string" || typeof value === "number") { + vars.push({ + name, + varType: "literal", + text: `${value}` + }); + continue; + } + if (typeof value === "object") { + try { + let separator = "|"; + if (typeof value.separator === "string") { + separator = value.separator; + } + + let items = arrayProp("any_of", value); + if (items) { + vars.push({ + name, + varType: "any_of_array", + items, + separator, + }); + continue; + } + + items = arrayProp("all_of", value); + if (items) { + vars.push({ + name, + varType: "all_of_array", + items, + separator, + }); + continue; + } + + if (typeof value.pattern === "string") { + let flags = ""; + if(typeof value.flags === "string") { + flags = value.flags; + } + + vars.push({ + name, + varType: "pattern", + pattern: new RegExp(value.pattern, flags), + }); + } + } catch (e) { + throw new Error(`failed to parse '${name}' variable: ${e}`); } } - } else { - throw new Error("Config file format error: no 'permit_query' property, or not an array."); } + return vars; +} + +export async function loadConfigFile(filePath: string): Promise { + const doc = yaml.load(await readFile(filePath, "utf8")) as { + queries_allow?: any[], + variables?: object, + labels_allow?: object, + }; + + if (typeof doc.variables !== "object") { + throw new Error("'variables' defined in config file, but not a dictionary."); + } + + if (doc.variables) { + config.variables = loadConfigFileVariables(doc.variables); + } + + if (typeof doc.labels_allow === "object") { + config.labelsAllow = []; + for(const [key, value] of Object.entries(doc.labels_allow)) { + if (typeof value === "boolean" && value) { + config.labelsAllow.push({ name: key }); + } else { + const values = arrayProp(key, doc.labels_allow)!; + config.labelsAllow.push({ name: key, values }); + } + } + } + + if (!Array.isArray(doc.queries_allow)) { + throw new Error("Config file format error: no 'queries_allow' property, or not an array."); + } + + config.queriesAllow = []; + for (const item of doc.queries_allow) { + if (typeof item === "string") { + config.queriesAllow.push(item); + } + } + + const queries = []; + for (const q of config.queriesAllow) { + queries.push(compileQuery(q, config.variables)); + } + config.queriesCompiled = queries; } await loadConfigFile(config.configFile); diff --git a/src/main.ts b/src/main.ts index 78a0ec7..994af96 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,53 +1,65 @@ -import { collectDefaultMetrics, Counter, Registry } from "prom-client"; +import { collectDefaultMetrics, Counter, Histogram, Registry } from "prom-client"; import { App, Request, Response } from '@tinyhttp/app' -import { config } from "./config.js"; +import { AnyVariable, config } from "./config.js"; import fetch from "node-fetch"; +import { compileQuery, queryToString } from "./query.js"; +import { validateQuery } from "./validator.js"; + const app = new App() const registry = new Registry(); collectDefaultMetrics({ register: registry }); -let rejectedCounter = new Counter({ +const rejectedCounter = new Counter({ name: "prom_policy_requests_rejected_count", help: "Number of rejected requests to datasource", labelNames: ["endpoint"], }); -let badRequestCounter = new Counter({ +const badRequestCounter = new Counter({ name: "prom_policy_bad_requests_count", help: "Number of bad requests to datasource", labelNames: ["endpoint"], }); -let requestsServedCounter = new Counter({ +const requestsServedCounter = new Counter({ name: "prom_policy_requests_served_count", help: "Number of successfully served requests to datasource", labelNames: ["endpoint"], }); + +const downstreamResponseTimeHistogram = new Histogram({ + name: "prom_policy_downstream_response_time", + help: "Histogram of downstream prometheus response times", + labelNames: ["endpoint"], +}); + registry.registerMetric(rejectedCounter); registry.registerMetric(badRequestCounter); registry.registerMetric(requestsServedCounter); +registry.registerMetric(downstreamResponseTimeHistogram); + +function first(strOrArr: T | T[] | undefined) { + if(Array.isArray(strOrArr)) { + return strOrArr[0]; + } + return strOrArr; +} function readParams(req: Request, params: string[]) { - let data: { [key: string]: string } = {}; + let data: Record = {}; if (req.headers["content-type"] === "application/x-www-form-urlencoded") { const bodyData = new URLSearchParams(String(req.read())); for(const p of params) { - const v = bodyData.get(p); - if(v) { - data[p] = v; + const v = bodyData.getAll(p); + if (v.length !== 0) { + data[p] = v.length === 1 ? v[0] : v; } } } else { - function first(strOrArr: T | T[] | undefined) { - if(Array.isArray(strOrArr)) { - return strOrArr[0]; - } - return strOrArr; - } for(const p of params) { - const v = first(req.query[p]); + const v = req.query[p]; if(v) { data[p] = v; } @@ -57,7 +69,33 @@ function readParams(req: Request, params: string[]) { return data; } -function validateQuery(query: string, req: Request, res: Response) { +function flattenRecordToArray(rec: Record): string[][] { + const result: string[][] = []; + for (const [k, va] of Object.entries(rec)) { + if (Array.isArray(va)) { + result.push(...va.map(v => [k, v])); + } else { + result.push([k, va]); + } + } + return result; +} + +// function flattenRecord(recs: Record[]): Record[] { +// const result: Record[] = []; +// for(const r of recs) { +// for (const [k, va] of Object.entries(r)) { +// if (Array.isArray(va)) { +// result.push(...va.map(v => ({ [k]: v }))); +// } else { +// result.push({ [k]: va }); +// } +// } +// } +// return result; +// } + +function validateQueryRequest(query: string | undefined, req: Request, res: Response) { if(!query) { badRequestCounter.inc({ endpoint: req.path }); res.statusCode = 400; @@ -69,7 +107,16 @@ function validateQuery(query: string, req: Request, res: Response) { return false; } - if(!config.permit_query.includes(query)) { + let allow = false; + for (const q of config.queriesCompiled) { + const { result } = validateQuery(query, q, config.variables); + if (result) { + allow = true; + break; + } + } + if (!allow) { + console.log(`Rejected query '${query}'`); rejectedCounter.inc({ endpoint: req.path }); res.statusCode = 403; res.json({ @@ -77,9 +124,8 @@ function validateQuery(query: string, req: Request, res: Response) { errorType: "access_denied", error: "you are not allowed to perform this query. (bonk!)", }); - return false; } - return true; + return allow; } app.all("/api/v1/query", async (req, res) => { @@ -87,21 +133,24 @@ app.all("/api/v1/query", async (req, res) => { console.log(data); - if(!validateQuery(data.query, req, res)) { - console.log("query was rejected."); + if(!validateQueryRequest(first(data.query), req, res)) { return; } - let promRes = await fetch(config.prometheusURL + "/api/v1/query", { + const end = downstreamResponseTimeHistogram.startTimer({ endpoint: req.path }); + + let promRes = await fetch(config.prometheusURL + req.path, { method: 'POST', headers:{ 'Content-Type': 'application/x-www-form-urlencoded' }, - body: new URLSearchParams(data), + body: new URLSearchParams(flattenRecordToArray(data)), }); + end(); + let promData = await promRes.text(); - console.log("prometheus response:", promData); + res.send(promData); requestsServedCounter.inc({ endpoint: req.path }); }); @@ -111,32 +160,164 @@ app.all("/api/v1/query_range", async (req, res) => { console.log(data); - if(!validateQuery(data.query, req, res)) { - console.log("query was rejected."); + if(!validateQueryRequest(first(data.query), req, res)) { return; } - let promRes = await fetch(config.prometheusURL + "/api/v1/query_range", { + const end = downstreamResponseTimeHistogram.startTimer({ endpoint: req.path }); + + let promRes = await fetch(config.prometheusURL + req.path, { method: 'POST', headers:{ 'Content-Type': 'application/x-www-form-urlencoded' }, - body: new URLSearchParams(data), + body: new URLSearchParams(flattenRecordToArray(data)), }); + end(); + let promData = await promRes.text(); - console.log("prometheus response:", promData); + res.send(promData); requestsServedCounter.inc({ endpoint: req.path }); }); -app.get("/api/v1/series", (req, res) => {}); -app.post("/api/v1/series", (req, res) => {}); +app.all("/api/v1/series", async (req, res) => { + const data = readParams(req, ["start", "end", "match[]", "limit"]); -app.get("/api/v1/labels", (req, res) => {}); -app.post("/api/v1/labels", (req, res) => {}); + console.log(data); -app.get("/api/v1/label//values", (req, res) => {}); + const end = downstreamResponseTimeHistogram.startTimer({ endpoint: req.path }); + + let promRes = await fetch(config.prometheusURL + req.path, { + method: 'POST', + headers:{ + 'Content-Type': 'application/x-www-form-urlencoded' + }, + body: new URLSearchParams(flattenRecordToArray(data)), + }); + + end(); + + let promData = await promRes.json() as { status: string, data: Record[], error?: string }; + + if (promData.status !== "success") { + res.statusCode = 502; + res.json({ + status: "error", + errorType: "downstream_error", + error: "downstream prometheus returned an error", + }); + return; + } + const allowedLabels = config.labelsAllow.map(l => l.name); + + // it's 1:30 am and oh girrrl this sucks + const series = new Map>( + promData.data.map(s => + Object.entries(s).filter(e => + allowedLabels.includes(e[0]) && + (config.labelsAllow.find(l => l.name === e[0])?.values ?? [e[1]]).includes(e[1]) + ) + ).map(Object.fromEntries).map(v => [Object.keys(v).join() + ":" + Object.values(v).join(), v]) + ); + + promData.data = Array.from(series.values()); + + res.send(promData); + requestsServedCounter.inc({ endpoint: req.path }); +}); + +app.all("/api/v1/labels", async (req, res) => { + const data = readParams(req, ["start", "end", "match[]", "limit"]); + + console.log(data); + + const end = downstreamResponseTimeHistogram.startTimer({ endpoint: req.path }); + + let promRes = await fetch(config.prometheusURL + req.path, { + method: 'POST', + headers:{ + 'Content-Type': 'application/x-www-form-urlencoded' + }, + body: new URLSearchParams(flattenRecordToArray(data)), + }); + + end(); + + let promData = await promRes.json() as { status: string, data: string[], error?: string }; + + if (promData.status !== "success") { + res.statusCode = 502; + res.json({ + status: "error", + errorType: "downstream_error", + error: "downstream prometheus returned an error", + }); + return; + } + const allowedLabels = config.labelsAllow.map(l => l.name); + + promData.data = promData.data.filter(l => allowedLabels.includes(l)); + + res.send(promData); + requestsServedCounter.inc({ endpoint: req.path }); +}); + +app.get("/api/v1/label/:label_name/values", async (req, res) => { + const label = req.params["label_name"]; + + if (!label) { + res.statusCode = 400; + res.json({ + status: "error", + errorType: "bad_data", + error: "no label name in url", + }); + return; + } + + const allowLabel = config.labelsAllow.find(l => l.name === label); + + if (!allowLabel) { + res.statusCode = 403; + res.json({ + status: "error", + errorType: "access_denied", + error: "you are not allowed to query this label. (bonk!)", + }); + return; + } + + const data = readParams(req, ["start", "end", "match[]", "limit"]); + + console.log(data); + + const end = downstreamResponseTimeHistogram.startTimer({ endpoint: req.path }); + + let promRes = await fetch(config.prometheusURL + req.path + "?" + new URLSearchParams(flattenRecordToArray(data)), { + method: 'GET', + }); + + end(); + + let promData = await promRes.json() as { status: string, data: string[], error?: string }; + + if (promData.status !== "success") { + res.statusCode = 502; + res.json({ + status: "error", + errorType: "downstream_error", + error: "downstream prometheus returned an error", + }); + return; + } + + promData.data = promData.data.filter(v => allowLabel.values ? allowLabel.values.includes(v) : true); + + res.send(promData); + requestsServedCounter.inc({ endpoint: req.path }); +}); app.get('/metrics', async (req, res) => { diff --git a/src/query.ts b/src/query.ts new file mode 100644 index 0000000..f37f029 --- /dev/null +++ b/src/query.ts @@ -0,0 +1,99 @@ +import { AnyVariable } from "./config.js"; + +interface QueryVariable { + name: string; + start: number; + part: number; +} + +export interface Query { + raw: string; + vars: QueryVariable[]; + parts: string[]; +} + +export function compileQuery(q: string, vars: AnyVariable[]): Query { + let queryVars: (QueryVariable | null)[] = []; + let queryParts: (string | null)[] = []; + + let lastIndex = 0; + let partStart = 0; + let partNum = 0; + + while (lastIndex = q.indexOf("$", lastIndex), lastIndex !== -1) { + const varName = (q.slice(lastIndex + 1).match(/([a-z0-9_-]*)/i) ?? [""])[0]; + + const variable = vars.find(v => v.name === varName); + if (!variable) { + throw new Error(`unknown variable '$${varName}' defined in query.`); + } + + queryParts.push(q.slice(partStart, lastIndex)); + + queryVars.push({ + name: varName, + start: lastIndex, + part: partNum + }); + + lastIndex += varName.length + 1; + partStart = lastIndex; + partNum += 1; + } + + queryParts.push(q.slice(partStart)); + + let partsRemoved = 0; + for (const [qVarIndex, qVar] of queryVars.entries()) { + if (qVar === null) continue; + + const v = vars.find(va => va.name === qVar.name)!; + if (v.varType === "literal") { + queryParts[qVar.part] += v.text + queryParts[qVar.part + 1]; + + queryParts[qVar.part + 1] = null; + + queryVars[qVarIndex] = null; + partsRemoved += 1; + } + + qVar.part -= partsRemoved; + } + + return { + raw: q, + vars: queryVars.filter(v => v !== null), + parts: queryParts.filter(p => p != null), + }; +} + +export function queryToString(query: Query, vars: AnyVariable[]) { + let output = ""; + for (const p of query.parts.keys()) { + output += query.parts[p]; + + if (p === query.parts.length - 1) { + break; + } + + const qvar = query.vars.find(v => v.part === p); + if (!qvar) { + throw new Error(`Not found QueryVariable of part with index ${p}`); + } + + const v = vars.find(v => v.name === qvar.name); + if (!v) { + throw new Error(`Not found Config Variable of part with index ${p}`); + } + + if (v.varType === "all_of_array" || v.varType === "any_of_array") { + output += v.items.join(v.separator); + } else if (v.varType === "pattern") { + output += "$" + v.name; + } else if (v.varType === "literal") { + output += v.text; + } + } + + return output; +} \ No newline at end of file diff --git a/src/validator.ts b/src/validator.ts new file mode 100644 index 0000000..7ab0f37 --- /dev/null +++ b/src/validator.ts @@ -0,0 +1,93 @@ +import { AnyVariable } from "./config.js"; +import { Query } from "./query.js"; + +export interface ValidationResult { + result: boolean; + message: string; + value?: string; +} + +export function validateQuery(input: string, query: Query, vars: AnyVariable[]): ValidationResult { + let start = 0; + for (const p of query.parts.keys()) { + const part = input.slice(start, start + query.parts[p].length); + + if (part !== query.parts[p]) { + return { + result: false, + message: `Parts with index '${p}' did not match.`, + value: part + } + } + + start += query.parts[p].length; + + const nextPartIndex = input.indexOf(query.parts[p + 1], start); + if (nextPartIndex === -1) { + if (p === query.parts.length - 1) { + return { + result: true, + message: `Validation successful.`, + } + } + return { + result: false, + message: `Next part (${p + 1}) not found.`, + } + } + + const qvar = query.vars.find(v => v.part === p); + if (!qvar) { + throw new Error(`Not found QueryVariable of part with index ${p}`); + } + + const v = vars.find(v => v.name === qvar.name); + if (!v) { + throw new Error(`Not found Config Variable of part with index ${p}`); + } + + const variableValue = input.slice(start, nextPartIndex); + + if (v.varType === "all_of_array" || v.varType === "any_of_array") { + const items = variableValue.split(v.separator); + if (v.varType === "all_of_array") { + if(v.items.find(i => !items.includes(i))) { + return { + result: false, + message: `Check '${v.varType}' failed for variable '${v.name}', part ${p}.`, + value: variableValue, + } + } + } else { + if(!v.items.find(i => items.includes(i))) { + return { + result: false, + message: `Check '${v.varType}' failed for variable '${v.name}', part ${p}.`, + value: variableValue, + } + } + } + } else if (v.varType === "pattern") { + const match = variableValue.match(v.pattern); + if (!match || !match[0]) { + return { + result: false, + message: `Check '${v.varType}' failed for variable '${v.name}', part ${p}.`, + value: variableValue, + } + } + } else if (v.varType === "literal") { + if (variableValue !== v.text) { + return { + result: false, + message: `Check '${v.varType}' failed for variable '${v.name}', part ${p}.`, + value: variableValue, + } + } + } + + start += variableValue.length; + } + + throw new Error(`Something is very broken. Double check your configuration.`); +} \ No newline at end of file