soffee/prometheus_policy_proxy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

i'm sorry for this garbage ah code

Browse source
This commit is contained in:
soffee 2025-04-11 02:00:58 +03:00
parent a0f765131e
commit 2ea075ab44
5 changed files with 613 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
config.example.yaml
View file

@ -0,0 +1,46 @@
labels_allow:
__name__:
- up
- messenger_dialog_info
- messenger_dialog_messages_count
- messenger_dialog_keywords_count
job:
- mtproto_exporter
instance:
- some_prometheus_instance:9669
peerId:
- "-9001234567890"
- "-9000987654321"
- "-9001122334455"
- "-9006677889900"
peerType: true
queries_allow:
- 1+1
- sum(increase(messenger_dialog_keywords_count{job="$job", peerId=~"$allowed_peers", peerId=~"$chat"}[30d])) by (keyword)
- >-
increase(messenger_dialog_messages_count{job="$job", peerId=~"$allowed_peers", peerId=~"$chat"}[30d]) *
on (peerId) group_right messenger_dialog_info{job="$job", peerId=~"$allowed_peers", peerId=~"$chat"}
- >-
sum(rate(messenger_dialog_keywords_count{job="$job", peerId=~"$allowed_peers", peerId=~"$chat"}[1h])
or (avg_over_time(messenger_dialog_keywords_count{job="$job", peerId=~"$allowed_peers", peerId=~"$chat"}[$__range]) * 0))
by (keyword) * 3600
variables:
job: mtproto_exporter
allowed_peers:
separator: "|"
all_of:
- "-9001234567890"
- "-9000987654321"
- "-9001122334455"
- "-9006677889900"
chat:
separator: "|"
any_of:
- "-9001234567890"
- "-9000987654321"
- "-9001122334455"
- "-9006677889900"
__range:
pattern: "^[0-9]+[ywdhms]?$"

170
src/config.ts
View file

@ -3,6 +3,7 @@ import { readFile } from "node:fs/promises";
import fs from "node:fs";
import cmdline from "command-line-args";
import yaml from "js-yaml";
import { compileQuery, Query } from "./query.js";
export interface Configuration {
bindHost: string;
@ -10,9 +11,49 @@ export interface Configuration {
configFile: string;
watchFile: boolean;
prometheusURL: string;
permit_query: string[];
queriesAllow: string[];
variables: AnyVariable[];
queriesCompiled: Query[];
labelsAllow: LabelAllow[];
}
interface LabelAllow {
name: string;
values?: string[];
}
interface BaseVariable<T extends string> {
name: string;
varType: T
}
export interface LiteralVariable extends BaseVariable<"literal"> {
text: string;
}
export function isAnyVariable(o: any): o is AnyVariable {
return (
typeof o === "object" &&
typeof o.name === "string" &&
typeof o.varType === "string"
);
}
interface ArrayLikeVariable<T extends string> extends BaseVariable<`${T}_array`> {
items: string[];
separator: string;
}
export type AnyOfItemsVariable = ArrayLikeVariable<"any_of">;
export type AllOfItemsVariable = ArrayLikeVariable<"all_of">;
export interface PatternVariable extends BaseVariable<"pattern"> {
pattern: RegExp;
}
export type AnyVariable = LiteralVariable | AnyOfItemsVariable | AllOfItemsVariable | PatternVariable;
const optionDefinitions: OptionDefinition[] = [
{ name: "bind-host", alias: "b", type: String, defaultValue: "0.0.0.0" },
{ name: "port", alias: "p", type: Number, defaultValue: 9091 },
@ -33,22 +74,129 @@ const config: Configuration = {
configFile: cli["config-file"],
watchFile: cli["watch-file"],
prometheusURL: cli["prometheus-url"],
permit_query: [],
queriesAllow: [],
variables: [],
queriesCompiled: [],
labelsAllow: [],
};
export async function loadConfigFile(filePath: string): Promise<void> {
const doc = yaml.load(await readFile(filePath, "utf8")) as { permit_query?: any[] };
function arrayProp(propName: string, target: any): string[] | undefined {
if (!(propName in target)) {
return;
}
if (!Array.isArray(target[propName])) {
throw new Error(`property '${propName}' should be an array.`);
}
const badElementIndex = target[propName].findIndex(v => typeof v !== "string" && typeof v !== "number");
if (badElementIndex !== -1) {
throw new Error(`all elements of '${propName}' should be strings or numbers. offending element index is ${badElementIndex}.`);
}
return target[propName];
}
if (typeof doc.permit_query === "object" && doc.permit_query.constructor.name === "Array") {
for (const item of doc.permit_query) {
if (typeof item === "string") {
config.permit_query.push(item);
// } else if (typeof item === "object") {
function loadConfigFileVariables(data: object): AnyVariable[] {
const vars: AnyVariable[] = [];
for (const [name, value] of Object.entries(data)) {
if (typeof value === "string" || typeof value === "number") {
vars.push({
name,
varType: "literal",
text: `${value}`
});
continue;
}
if (typeof value === "object") {
try {
let separator = "|";
if (typeof value.separator === "string") {
separator = value.separator;
}
let items = arrayProp("any_of", value);
if (items) {
vars.push({
name,
varType: "any_of_array",
items,
separator,
});
continue;
}
items = arrayProp("all_of", value);
if (items) {
vars.push({
name,
varType: "all_of_array",
items,
separator,
});
continue;
}
if (typeof value.pattern === "string") {
let flags = "";
if(typeof value.flags === "string") {
flags = value.flags;
}
vars.push({
name,
varType: "pattern",
pattern: new RegExp(value.pattern, flags),
});
}
} catch (e) {
throw new Error(`failed to parse '${name}' variable: ${e}`);
}
}
} else {
throw new Error("Config file format error: no 'permit_query' property, or not an array.");
}
return vars;
}
export async function loadConfigFile(filePath: string): Promise<void> {
const doc = yaml.load(await readFile(filePath, "utf8")) as {
queries_allow?: any[],
variables?: object,
labels_allow?: object,
};
if (typeof doc.variables !== "object") {
throw new Error("'variables' defined in config file, but not a dictionary.");
}
if (doc.variables) {
config.variables = loadConfigFileVariables(doc.variables);
}
if (typeof doc.labels_allow === "object") {
config.labelsAllow = [];
for(const [key, value] of Object.entries(doc.labels_allow)) {
if (typeof value === "boolean" && value) {
config.labelsAllow.push({ name: key });
} else {
const values = arrayProp(key, doc.labels_allow)!;
config.labelsAllow.push({ name: key, values });
}
}
}
if (!Array.isArray(doc.queries_allow)) {
throw new Error("Config file format error: no 'queries_allow' property, or not an array.");
}
config.queriesAllow = [];
for (const item of doc.queries_allow) {
if (typeof item === "string") {
config.queriesAllow.push(item);
}
}
const queries = [];
for (const q of config.queriesAllow) {
queries.push(compileQuery(q, config.variables));
}
config.queriesCompiled = queries;
}
await loadConfigFile(config.configFile);

251
src/main.ts
View file

@ -1,53 +1,65 @@
import { collectDefaultMetrics, Counter, Registry } from "prom-client";
import { collectDefaultMetrics, Counter, Histogram, Registry } from "prom-client";
import { App, Request, Response } from '@tinyhttp/app'
import { config } from "./config.js";
import { AnyVariable, config } from "./config.js";
import fetch from "node-fetch";
import { compileQuery, queryToString } from "./query.js";
import { validateQuery } from "./validator.js";
const app = new App()
const registry = new Registry();
collectDefaultMetrics({ register: registry });
let rejectedCounter = new Counter({
const rejectedCounter = new Counter({
name: "prom_policy_requests_rejected_count",
help: "Number of rejected requests to datasource",
labelNames: ["endpoint"],
});
let badRequestCounter = new Counter({
const badRequestCounter = new Counter({
name: "prom_policy_bad_requests_count",
help: "Number of bad requests to datasource",
labelNames: ["endpoint"],
});
let requestsServedCounter = new Counter({
const requestsServedCounter = new Counter({
name: "prom_policy_requests_served_count",
help: "Number of successfully served requests to datasource",
labelNames: ["endpoint"],
});
const downstreamResponseTimeHistogram = new Histogram({
name: "prom_policy_downstream_response_time",
help: "Histogram of downstream prometheus response times",
labelNames: ["endpoint"],
});
registry.registerMetric(rejectedCounter);
registry.registerMetric(badRequestCounter);
registry.registerMetric(requestsServedCounter);
registry.registerMetric(downstreamResponseTimeHistogram);
function first<T>(strOrArr: T | T[] | undefined) {
if(Array.isArray(strOrArr)) {
return strOrArr[0];
}
return strOrArr;
}
function readParams(req: Request, params: string[]) {
let data: { [key: string]: string } = {};
let data: Record<string, string[] | string> = {};
if (req.headers["content-type"] === "application/x-www-form-urlencoded") {
const bodyData = new URLSearchParams(String(req.read()));
for(const p of params) {
const v = bodyData.get(p);
if(v) {
data[p] = v;
const v = bodyData.getAll(p);
if (v.length !== 0) {
data[p] = v.length === 1 ? v[0] : v;
}
}
} else {
function first<T>(strOrArr: T | T[] | undefined) {
if(Array.isArray(strOrArr)) {
return strOrArr[0];
}
return strOrArr;
}
for(const p of params) {
const v = first(req.query[p]);
const v = req.query[p];
if(v) {
data[p] = v;
}
@ -57,7 +69,33 @@ function readParams(req: Request, params: string[]) {
return data;
}
function validateQuery(query: string, req: Request, res: Response) {
function flattenRecordToArray(rec: Record<string, string | string[]>): string[][] {
const result: string[][] = [];
for (const [k, va] of Object.entries<string | string[]>(rec)) {
if (Array.isArray(va)) {
result.push(...va.map(v => [k, v]));
} else {
result.push([k, va]);
}
}
return result;
}
// function flattenRecord<K extends string | number | symbol, V>(recs: Record<K, V | V[]>[]): Record<K, V>[] {
// const result: Record<K, V>[] = [];
// for(const r of recs) {
// for (const [k, va] of Object.entries<V | V[]>(r)) {
// if (Array.isArray(va)) {
// result.push(...va.map(v => ({ [k]: v })));
// } else {
// result.push({ [k]: va });
// }
// }
// }
// return result;
// }
function validateQueryRequest(query: string | undefined, req: Request, res: Response) {
if(!query) {
badRequestCounter.inc({ endpoint: req.path });
res.statusCode = 400;
@ -69,7 +107,16 @@ function validateQuery(query: string, req: Request, res: Response) {
return false;
}
if(!config.permit_query.includes(query)) {
let allow = false;
for (const q of config.queriesCompiled) {
const { result } = validateQuery(query, q, config.variables);
if (result) {
allow = true;
break;
}
}
if (!allow) {
console.log(`Rejected query '${query}'`);
rejectedCounter.inc({ endpoint: req.path });
res.statusCode = 403;
res.json({
@ -77,9 +124,8 @@ function validateQuery(query: string, req: Request, res: Response) {
errorType: "access_denied",
error: "you are not allowed to perform this query. (bonk!)",
});
return false;
}
return true;
return allow;
}
app.all("/api/v1/query", async (req, res) => {
@ -87,21 +133,24 @@ app.all("/api/v1/query", async (req, res) => {
console.log(data);
if(!validateQuery(data.query, req, res)) {
console.log("query was rejected.");
if(!validateQueryRequest(first(data.query), req, res)) {
return;
}
let promRes = await fetch(config.prometheusURL + "/api/v1/query", {
const end = downstreamResponseTimeHistogram.startTimer({ endpoint: req.path });
let promRes = await fetch(config.prometheusURL + req.path, {
method: 'POST',
headers:{
'Content-Type': 'application/x-www-form-urlencoded'
},
body: new URLSearchParams(data),
body: new URLSearchParams(flattenRecordToArray(data)),
});
end();
let promData = await promRes.text();
console.log("prometheus response:", promData);
res.send(promData);
requestsServedCounter.inc({ endpoint: req.path });
});
@ -111,32 +160,164 @@ app.all("/api/v1/query_range", async (req, res) => {
console.log(data);
if(!validateQuery(data.query, req, res)) {
console.log("query was rejected.");
if(!validateQueryRequest(first(data.query), req, res)) {
return;
}
let promRes = await fetch(config.prometheusURL + "/api/v1/query_range", {
const end = downstreamResponseTimeHistogram.startTimer({ endpoint: req.path });
let promRes = await fetch(config.prometheusURL + req.path, {
method: 'POST',
headers:{
'Content-Type': 'application/x-www-form-urlencoded'
},
body: new URLSearchParams(data),
body: new URLSearchParams(flattenRecordToArray(data)),
});
end();
let promData = await promRes.text();
console.log("prometheus response:", promData);
res.send(promData);
requestsServedCounter.inc({ endpoint: req.path });
});
app.get("/api/v1/series", (req, res) => {});
app.post("/api/v1/series", (req, res) => {});
app.all("/api/v1/series", async (req, res) => {
const data = readParams(req, ["start", "end", "match[]", "limit"]);
app.get("/api/v1/labels", (req, res) => {});
app.post("/api/v1/labels", (req, res) => {});
console.log(data);
app.get("/api/v1/label/<label_name>/values", (req, res) => {});
const end = downstreamResponseTimeHistogram.startTimer({ endpoint: req.path });
let promRes = await fetch(config.prometheusURL + req.path, {
method: 'POST',
headers:{
'Content-Type': 'application/x-www-form-urlencoded'
},
body: new URLSearchParams(flattenRecordToArray(data)),
});
end();
let promData = await promRes.json() as { status: string, data: Record<string, string>[], error?: string };
if (promData.status !== "success") {
res.statusCode = 502;
res.json({
status: "error",
errorType: "downstream_error",
error: "downstream prometheus returned an error",
});
return;
}
const allowedLabels = config.labelsAllow.map(l => l.name);
// it's 1:30 am and oh girrrl this sucks
const series = new Map<string, Record<string, string>>(
promData.data.map(s =>
Object.entries(s).filter(e =>
allowedLabels.includes(e[0]) &&
(config.labelsAllow.find(l => l.name === e[0])?.values ?? [e[1]]).includes(e[1])
)
).map(Object.fromEntries).map(v => [Object.keys(v).join() + ":" + Object.values(v).join(), v])
);
promData.data = Array.from(series.values());
res.send(promData);
requestsServedCounter.inc({ endpoint: req.path });
});
app.all("/api/v1/labels", async (req, res) => {
const data = readParams(req, ["start", "end", "match[]", "limit"]);
console.log(data);
const end = downstreamResponseTimeHistogram.startTimer({ endpoint: req.path });
let promRes = await fetch(config.prometheusURL + req.path, {
method: 'POST',
headers:{
'Content-Type': 'application/x-www-form-urlencoded'
},
body: new URLSearchParams(flattenRecordToArray(data)),
});
end();
let promData = await promRes.json() as { status: string, data: string[], error?: string };
if (promData.status !== "success") {
res.statusCode = 502;
res.json({
status: "error",
errorType: "downstream_error",
error: "downstream prometheus returned an error",
});
return;
}
const allowedLabels = config.labelsAllow.map(l => l.name);
promData.data = promData.data.filter(l => allowedLabels.includes(l));
res.send(promData);
requestsServedCounter.inc({ endpoint: req.path });
});
app.get("/api/v1/label/:label_name/values", async (req, res) => {
const label = req.params["label_name"];
if (!label) {
res.statusCode = 400;
res.json({
status: "error",
errorType: "bad_data",
error: "no label name in url",
});
return;
}
const allowLabel = config.labelsAllow.find(l => l.name === label);
if (!allowLabel) {
res.statusCode = 403;
res.json({
status: "error",
errorType: "access_denied",
error: "you are not allowed to query this label. (bonk!)",
});
return;
}
const data = readParams(req, ["start", "end", "match[]", "limit"]);
console.log(data);
const end = downstreamResponseTimeHistogram.startTimer({ endpoint: req.path });
let promRes = await fetch(config.prometheusURL + req.path + "?" + new URLSearchParams(flattenRecordToArray(data)), {
method: 'GET',
});
end();
let promData = await promRes.json() as { status: string, data: string[], error?: string };
if (promData.status !== "success") {
res.statusCode = 502;
res.json({
status: "error",
errorType: "downstream_error",
error: "downstream prometheus returned an error",
});
return;
}
promData.data = promData.data.filter(v => allowLabel.values ? allowLabel.values.includes(v) : true);
res.send(promData);
requestsServedCounter.inc({ endpoint: req.path });
});
app.get('/metrics', async (req, res) => {

99
src/query.ts Normal file
View file

@ -0,0 +1,99 @@
import { AnyVariable } from "./config.js";
interface QueryVariable {
name: string;
start: number;
part: number;
}
export interface Query {
raw: string;
vars: QueryVariable[];
parts: string[];
}
export function compileQuery(q: string, vars: AnyVariable[]): Query {
let queryVars: (QueryVariable | null)[] = [];
let queryParts: (string | null)[] = [];
let lastIndex = 0;
let partStart = 0;
let partNum = 0;
while (lastIndex = q.indexOf("$", lastIndex), lastIndex !== -1) {
const varName = (q.slice(lastIndex + 1).match(/([a-z0-9_-]*)/i) ?? [""])[0];
const variable = vars.find(v => v.name === varName);
if (!variable) {
throw new Error(`unknown variable '$${varName}' defined in query.`);
}
queryParts.push(q.slice(partStart, lastIndex));
queryVars.push({
name: varName,
start: lastIndex,
part: partNum
});
lastIndex += varName.length + 1;
partStart = lastIndex;
partNum += 1;
}
queryParts.push(q.slice(partStart));
let partsRemoved = 0;
for (const [qVarIndex, qVar] of queryVars.entries()) {
if (qVar === null) continue;
const v = vars.find(va => va.name === qVar.name)!;
if (v.varType === "literal") {
queryParts[qVar.part] += v.text + queryParts[qVar.part + 1];
queryParts[qVar.part + 1] = null;
queryVars[qVarIndex] = null;
partsRemoved += 1;
}
qVar.part -= partsRemoved;
}
return {
raw: q,
vars: queryVars.filter(v => v !== null),
parts: queryParts.filter(p => p != null),
};
}
export function queryToString(query: Query, vars: AnyVariable[]) {
let output = "";
for (const p of query.parts.keys()) {
output += query.parts[p];
if (p === query.parts.length - 1) {
break;
}
const qvar = query.vars.find(v => v.part === p);
if (!qvar) {
throw new Error(`Not found QueryVariable of part with index ${p}`);
}
const v = vars.find(v => v.name === qvar.name);
if (!v) {
throw new Error(`Not found Config Variable of part with index ${p}`);
}
if (v.varType === "all_of_array" || v.varType === "any_of_array") {
output += v.items.join(v.separator);
} else if (v.varType === "pattern") {
output += "$" + v.name;
} else if (v.varType === "literal") {
output += v.text;
}
}
return output;
}

93
src/validator.ts Normal file
View file

@ -0,0 +1,93 @@
import { AnyVariable } from "./config.js";
import { Query } from "./query.js";
export interface ValidationResult {
result: boolean;
message: string;
value?: string;
}
export function validateQuery(input: string, query: Query, vars: AnyVariable[]): ValidationResult {
let start = 0;
for (const p of query.parts.keys()) {
const part = input.slice(start, start + query.parts[p].length);
if (part !== query.parts[p]) {
return {
result: false,
message: `Parts with index '${p}' did not match.`,
value: part
}
}
start += query.parts[p].length;
const nextPartIndex = input.indexOf(query.parts[p + 1], start);
if (nextPartIndex === -1) {
if (p === query.parts.length - 1) {
return {
result: true,
message: `Validation successful.`,
}
}
return {
result: false,
message: `Next part (${p + 1}) not found.`,
}
}
const qvar = query.vars.find(v => v.part === p);
if (!qvar) {
throw new Error(`Not found QueryVariable of part with index ${p}`);
}
const v = vars.find(v => v.name === qvar.name);
if (!v) {
throw new Error(`Not found Config Variable of part with index ${p}`);
}
const variableValue = input.slice(start, nextPartIndex);
if (v.varType === "all_of_array" || v.varType === "any_of_array") {
const items = variableValue.split(v.separator);
if (v.varType === "all_of_array") {
if(v.items.find(i => !items.includes(i))) {
return {
result: false,
message: `Check '${v.varType}' failed for variable '${v.name}', part ${p}.`,
value: variableValue,
}
}
} else {
if(!v.items.find(i => items.includes(i))) {
return {
result: false,
message: `Check '${v.varType}' failed for variable '${v.name}', part ${p}.`,
value: variableValue,
}
}
}
} else if (v.varType === "pattern") {
const match = variableValue.match(v.pattern);
if (!match || !match[0]) {
return {
result: false,
message: `Check '${v.varType}' failed for variable '${v.name}', part ${p}.`,
value: variableValue,
}
}
} else if (v.varType === "literal") {
if (variableValue !== v.text) {
return {
result: false,
message: `Check '${v.varType}' failed for variable '${v.name}', part ${p}.`,
value: variableValue,
}
}
}
start += variableValue.length;
}
throw new Error(`Something is very broken. Double check your configuration.`);
}