Watch
1
0
Fork
You've already forked cloudflare-gateway-pihole-scripts
0
mirror of https://github.com/mrrfv/cloudflare-gateway-pihole-scripts.git synced 2026-11-16 19:37:01 +00:00
cloudflare-gateway-pihole-s.../lib/api.js
Ben Sykes 952c9fe370 Apply defrag additions before removals
Each touched list fired a single patch call in whatever order it was
first encountered during compaction, which could remove entries from a
donor list before the corresponding addition to its destination list
had been applied - leaving a domain blocked by nothing in between.

A list is only ever a source or a destination within one compaction
pass, never both, so split the patch loop into two passes: apply every
addition first, then every removal. This also updates lists in
ascending chunk order within each pass instead of a jumbled order.
2026-08-29 16:06:51 -07:00

513 lines
20 KiB
JavaScript

import { BLOCK_PAGE_ENABLED, DEBUG, LIST_ITEM_SIZE } from "./constants.js";
import { requestGateway } from "./helpers.js";
const NOW_STR = new Date().toISOString();
/**
* Gets Zero Trust lists.
*
* API docs: https://developers.cloudflare.com/api/operations/zero-trust-lists-list-zero-trust-lists
* @returns {Promise<Object>}
*/
export const getZeroTrustLists = () =>
requestGateway("/lists", {
method: "GET",
});
/**
* Gets Zero Trust list items
*
* API docs: https://developers.cloudflare.com/api/operations/zero-trust-lists-zero-trust-list-items
* @param {string} id The id of the list.
* @returns {Promise<Object>}
*/
const getZeroTrustListItems = (id) =>
requestGateway(`/lists/${id}/items?per_page=${LIST_ITEM_SIZE}`, {
method: "GET",
});
/**
* Creates a Zero Trust list.
*
* API docs: https://developers.cloudflare.com/api/operations/zero-trust-lists-create-zero-trust-list
* @param {string} name The name of the list.
* @param {Object[]} items The domains in the list.
* @param {string} items[].value The domain of an entry.
* @returns {Promise}
*/
const createZeroTrustList = (name, items) =>
requestGateway(`/lists`, {
method: "POST",
body: JSON.stringify({
name,
type: "DOMAIN",
items,
}),
});
/**
* Patches an existing list. Remove/append entries to the list.
*
* API docs: https://developers.cloudflare.com/api/operations/zero-trust-lists-patch-zero-trust-list
* @param {string} listId The ID of the list to patch
* @param {Object} patch The changes to make
* @param {string[]} patch.remove A list of the item values you want to remove.
* @param {Object[]} patch.append Items to add to the list.
* @param {string} patch.append[].value The domain of an entry.
* @returns
*/
const patchExistingList = (listId, patch) =>
requestGateway(`/lists/${listId}`, {
method: "PATCH",
body: JSON.stringify(patch),
});
/**
* Synchronize Zero Trust lists.
* Inspects existing lists starting with "CGPS List"
* Compares the entries in the lists with the desired domains in the items.
* Removes any entries in the lists that are not in the items.
* Adds any entries that are in the items and not in the lists.
* Uses available capacity in existing lists prior to creating a new list.
* @param {string[]} items The domains.
*/
export const synchronizeZeroTrustLists = async (items) => {
const itemSet = new Set(items);
console.log("Checking existing lists...");
const { result: lists } = await getZeroTrustLists();
const cgpsLists = lists?.filter(({ name }) => name.startsWith("CGPS List")) || [];
console.log(`Found ${cgpsLists.length} existing lists. Calculating diffs...`);
const domainsByList = {};
// Do this sequentially to avoid rate-limits
for (const list of cgpsLists) {
const { result: listItems, result_info } = await getZeroTrustListItems(list.id);
if (result_info.total_count > LIST_ITEM_SIZE) {
console.log(`List ${list.name} contains more entries that LIST_ITEM_SIZE. Checking only the first ${LIST_ITEM_SIZE} entires. You may want to delete this list and recreate using the same size limit.`);
}
domainsByList[list.id] = listItems?.map(item => item.value) || [];
}
// Extract all the list entries into a map, keyed by domain, pointing to the list.
const existingDomains = Object.fromEntries(
Object.entries(domainsByList).flatMap(([id, domains]) => domains.map(d => [d, id]))
);
// Create a list of entries to remove.
// Iterate the existing list(s) removing anything that's in the new list.
// Resulting in entries that are in the existing list(s) and not in the new list.
const toRemove = Object.fromEntries(
Object.entries(existingDomains).filter(([domain]) => !itemSet.has(domain))
);
// Create a list of entries to add.
// Iterate the new list keeping only entries not in the existing list(s).
// Resulting in entries that need to be added.
const toAdd = items.filter(domain => !existingDomains[domain]);
console.log(`${Object.keys(toRemove).length} removals, ${toAdd.length} additions to make`);
// Group the removals by list id, so we can make a patch request.
const removalPatches = Object.entries(toRemove).reduce((acc, [domain, listId]) => {
acc[listId] = acc[listId] || { remove: [] };
acc[listId].remove.push(domain);
return acc;
}, {});
// Fill any "gaps" in the lists made by the removals with any additions.
// If we can fit all the additions into the same lists that we're processing removals
// we can minimize the number of lists that need to be edited.
const patches = Object.fromEntries(
Object.entries(removalPatches).map(([listId, patch]) => {
// Work out how much "space" is in the list by looking at
// how many entries there were and how many we're removing.
const spaceInList = LIST_ITEM_SIZE - (domainsByList[listId].length - patch.remove.length);
// Take upto spaceInList entries from the additions into this list.
// Use the current timestamp as the description to track when we first see this domain.
// This can be used to defragment the lists later and consolidate more stable entries.
const append = Array(spaceInList)
.fill(0)
.map(() => toAdd.shift())
.filter(Boolean)
.map(domain => ({ value: domain, description: NOW_STR }));
return [listId, { ...patch, append }];
})
);
// Are there any more appends remaining?
if (toAdd.length) {
// Is there any space in any existing lists, other than those we're already patching?
const unpatchedListIds = Object.keys(domainsByList).filter(listId => !patches[listId]);
unpatchedListIds.forEach(listId => {
const spaceInList = LIST_ITEM_SIZE - domainsByList[listId].length;
if (spaceInList > 0) {
// Take upto spaceInList entries from the additions into this list.
const append = Array(spaceInList)
.fill(0)
.map(() => toAdd.shift())
.filter(Boolean)
.map(domain => ({ value: domain, description: NOW_STR }));
// Add this list edit to the patches
if (append.length) {
patches[listId] = { append };
}
}
});
}
// Process all the patches. Sequentially to avoid rate limits.
for(const [listId, patch] of Object.entries(patches)) {
const appends = !!patch.append ? patch.append.length : 0;
const removals = !!patch.remove ? patch.remove.length : 0;
console.log(`Updating list "${cgpsLists.find(list => list.id === listId).name}"${appends ? `, ${appends} additions` : ''}${removals ? `, ${removals} removals` : ''}`);
await patchExistingList(listId, patch);
}
// Are there any more appends remaining?
if (toAdd.length) {
// We'll need to create new list(s)
const nextListNumber = Math.max(0, ...cgpsLists.map(list => parseInt(list.name.replace('CGPS List - Chunk ', ''))).filter(x => Number.isInteger(x))) + 1;
await createZeroTrustListsOneByOne(toAdd, nextListNumber);
}
};
/**
* Defragment Zero Trust lists.
* Inspects existing lists starting with "CGPS List - Chunk <number>"
* Compacts them by pulling entries out of the back of the highest-numbered
* lists to fill any spare capacity in the lowest-numbered lists.
* This only touches lists that actually have spare capacity, or are being
* drained to fill one - a list that's already full is left completely alone.
* That keeps a single early removal from cascading into edits on every list
* that follows it: we don't require entries to sit in any particular global
* order, so filling a gap never requires reshuffling everything after it.
* @returns {Promise<Object>} A object that include the now empty and non-empty lists
*/
export const defragmentZeroTrustLists = async () => {
console.log("Checking existing lists...");
const { result: lists } = await getZeroTrustLists();
const cgpsLists = lists?.filter(({ name }) => name.startsWith("CGPS List - Chunk ")) || [];
console.log(`Found ${cgpsLists.length} existing lists. Downloading...`);
// Sort the lists by the natural number order in the name
cgpsLists.sort((a, b) => {
const aNum = parseInt(a.name.replace("CGPS List - Chunk ", ""));
const bNum = parseInt(b.name.replace("CGPS List - Chunk ", ""));
return aNum - bNum;
});
// Fetch all the items in the lists, keeping them grouped by the list they
// came from rather than merging into one big sorted sequence.
const entriesByList = new Map();
let allEntriesCount = 0;
for (const list of cgpsLists) {
const { result: listItems } = await getZeroTrustListItems(list.id);
// Ensure the description is a valid timestamp, or set it to the current time.
// We use the description as the list addition time because the API does not allow setting the created_at time.
const items = listItems?.map(item => ({
value: item.value,
description: isNaN(new Date(item.description)) ? NOW_STR : item.description,
})) || [];
entriesByList.set(list.id, items);
allEntriesCount += items.length;
}
console.log(`Found ${allEntriesCount} entries in ${cgpsLists.length} lists`);
// Compact in place: walk from the front looking for lists with spare
// capacity ("deficits"), and fill each one using entries pulled off the
// back of the highest-numbered lists that still have entries. A list keeps
// its number and its existing entries untouched unless it's the one being
// topped up or the one being drained.
const patches = {};
const getPatch = (listId) => {
if (!patches[listId]) patches[listId] = { append: [], remove: [] };
return patches[listId];
};
let entriesMoved = 0;
let lowIndex = 0;
let highIndex = cgpsLists.length - 1;
while (lowIndex < highIndex) {
const lowList = cgpsLists[lowIndex];
const lowEntries = entriesByList.get(lowList.id);
const deficit = LIST_ITEM_SIZE - lowEntries.length;
if (deficit <= 0) {
lowIndex++;
continue;
}
const highList = cgpsLists[highIndex];
const highEntries = entriesByList.get(highList.id);
if (highEntries.length === 0) {
highIndex--;
continue;
}
const moveCount = Math.min(deficit, highEntries.length);
const moved = highEntries.splice(highEntries.length - moveCount, moveCount);
lowEntries.push(...moved);
entriesMoved += moveCount;
for (const entry of moved) {
getPatch(lowList.id).append.push(entry);
getPatch(highList.id).remove.push(entry.value);
}
}
console.log(`Found ${Object.keys(patches).length} patches to make, moving ${entriesMoved} entries...`);
// Apply every addition before any removal. A list is only ever a source or
// a destination within a single compaction pass, never both, so doing all
// the appends first guarantees each moved entry is safely living in its new
// list before it's taken out of its old one - a domain blocked by two lists
// for a moment is harmless, but a domain blocked by neither is not.
for (const list of cgpsLists) {
const appends = patches[list.id]?.append;
if (!appends?.length) continue;
console.log(`Updating list "${list.name}", ${appends.length} additions`);
await patchExistingList(list.id, { append: appends });
}
for (const list of cgpsLists) {
const removals = patches[list.id]?.remove;
if (!removals?.length) continue;
console.log(`Updating list "${list.name}", ${removals.length} removals`);
await patchExistingList(list.id, { remove: removals });
}
// Any lists left with no entries are now empty and can be deleted
const emptyLists = cgpsLists.filter(list => entriesByList.get(list.id).length === 0);
// Gather the non-empty lists, using the original list not just the chunked ones
// This is important to capture any manually created lists starting with "CGPS List"
// and not just the ones created by this script
const nonEmptyLists = lists.filter(list => !emptyLists.some(emptyList => emptyList.id === list.id));
return {
emptyLists,
nonEmptyLists,
stats: {
assignedLists: cgpsLists.length - emptyLists.length,
emptyLists: emptyLists.length,
nonEmptyLists: nonEmptyLists.length,
entriesToMove: entriesMoved,
patches: Object.keys(patches).length,
allEntries: allEntriesCount,
chunks: cgpsLists.length,
}
};
}
/**
* Creates Zero Trust lists sequentially.
* @param {string[]} items The domains.
* @param {Number} [startingListNumber] The chunk number to start from when naming lists.
*/
export const createZeroTrustListsOneByOne = async (items, startingListNumber = 1) => {
let totalListNumber = Math.ceil(items.length / LIST_ITEM_SIZE);
for (let i = 0, listNumber = startingListNumber; i < items.length; i += LIST_ITEM_SIZE) {
// We use the description as the list addition time because the API does not allow setting the created_at time.
const chunk = items
.slice(i, i + LIST_ITEM_SIZE)
.map((item) => ({ value: item, description: NOW_STR }));
const listName = `CGPS List - Chunk ${listNumber}`;
try {
await createZeroTrustList(listName, chunk);
totalListNumber--;
listNumber++;
console.log(`Created "${listName}" list - ${totalListNumber} left`);
} catch (err) {
console.error(`Could not create "${listName}" - ${err.toString()}`);
throw err;
}
}
};
/**
* Deletes a Zero Trust list.
*
* API docs: https://developers.cloudflare.com/api/operations/zero-trust-lists-delete-zero-trust-list
* @param {number} id The ID of the list.
* @returns {Promise<any>}
*/
const deleteZeroTrustList = (id) =>
requestGateway(`/lists/${id}`, { method: "DELETE" });
/**
* Deletes Zero Trust lists sequentially.
* @param {Object[]} lists The lists to be deleted.
* @param {number} lists[].id The ID of a list.
* @param {string} lists[].name The name of a list.
*/
export const deleteZeroTrustListsOneByOne = async (lists) => {
let totalListNumber = lists.length;
for (const { id, name } of lists) {
try {
await deleteZeroTrustList(id);
totalListNumber--;
console.log(`Deleted ${name} list - ${totalListNumber} left`);
} catch (err) {
console.error(`Could not delete ${name} - ${err.toString()}`);
throw err;
}
}
};
/**
* Gets Zero Trust rules.
*
* API docs: https://developers.cloudflare.com/api/operations/zero-trust-gateway-rules-list-zero-trust-gateway-rules
* @returns {Promise<Object>}
*/
export const getZeroTrustRules = () =>
requestGateway("/rules", { method: "GET" });
/**
* Upserts a Zero Trust rule.
* If a rule with the same name exists, will update it. Otherwise create a new rule.
* @param {string} wirefilterExpression The expression to be used for the rule.
* @param {string} name The name of the rule.
* @param {string[]} filters The filters to be used for the rule. Default is ["dns"]. Possible values are ["dns", "http", "l4", "egress"].
* @returns {Promise<Object>}
*/
export const upsertZeroTrustRule = async (wirefilterExpression, name = "CGPS Filter Lists", filters = ["dns"]) => {
const { result: existingRules} = await getZeroTrustRules();
const existingRule = existingRules.find(rule => rule.name === name);
if (existingRule) {
if (DEBUG) console.log(`Found "${existingRule.name}" in rules, updating...`);
return updateZeroTrustRule(existingRule.id, wirefilterExpression, name, filters);
}
if (DEBUG) console.log(`No existing rule named "${existingRule.name}", creating...`);
return createZeroTrustRule(wirefilterExpression, name, filters);
}
/**
* Creates a Zero Trust rule.
*
* API docs: https://developers.cloudflare.com/api/operations/zero-trust-gateway-rules-create-zero-trust-gateway-rule
* @param {string} wirefilterExpression The expression to be used for the rule.
* @param {string} name The name of the rule.
* @param {string[]} filters The filters to be used for the rule. Default is ["dns"]. Possible values are ["dns", "http", "l4", "egress"].
* @returns {Promise<Object>}
*/
export const createZeroTrustRule = async (wirefilterExpression, name = "CGPS Filter Lists", filters = ["dns"]) => {
try {
await requestGateway("/rules", {
method: "POST",
body: JSON.stringify({
name,
description:
"Filter lists created by Cloudflare Gateway Pi-hole Scripts. Avoid editing this rule. Changing the name of this rule will break the script.",
enabled: true,
action: "block",
rule_settings: { "block_page_enabled": BLOCK_PAGE_ENABLED, "block_reason": "Blocked by CGPS, check your filter lists if this was a mistake." },
filters,
traffic: wirefilterExpression,
}),
});
console.log("Created rule successfully");
} catch (err) {
console.error(`Error occurred while creating rule - ${err.toString()}`);
throw err;
}
};
/**
* Updates a Zero Trust rule.
*
* API docs: https://developers.cloudflare.com/api/operations/zero-trust-gateway-rules-update-zero-trust-gateway-rule
* @param {number} id The ID of the rule to be updated.
* @param {string} wirefilterExpression The expression to be used for the rule.
* @param {string} name The name of the rule.
* @param {string[]} filters The filters to be used for the rule.
* @returns {Promise<Object>}
*/
export const updateZeroTrustRule = async (id, wirefilterExpression, name = "CGPS Filter Lists", filters = ["dns"]) => {
try {
await requestGateway(`/rules/${id}`, {
method: "PUT",
body: JSON.stringify({
// Name and action are required fields, even if they haven't changed.
// And enabled must always be set to true, otherwise the rule will be disabled if omitted.
name,
description:
"Filter lists created by Cloudflare Gateway Pi-hole Scripts. Avoid editing this rule. Changing the name of this rule will break the script.",
action: "block",
enabled: true,
rule_settings: { "block_page_enabled": BLOCK_PAGE_ENABLED, "block_reason": "Blocked by CGPS, check your filter lists if this was a mistake." },
filters,
traffic: wirefilterExpression,
}),
});
console.log("Updated existing rule successfully");
} catch (err) {
console.error(`Error occurred while updating rule - ${err.toString()}`);
throw err;
}
};
/**
* Deletes a Zero Trust rule.
*
* API docs: https://developers.cloudflare.com/api/operations/zero-trust-gateway-rules-delete-zero-trust-gateway-rule
* @param {number} id The ID of the rule to be deleted.
* @returns {Promise<Object>}
*/
export const deleteZeroTrustRule = async (id) => {
try {
await requestGateway(`/rules/${id}`, {
method: "DELETE",
});
console.log("Deleted rule successfully");
} catch (err) {
console.error(`Error occurred while deleting rule - ${err.toString()}`);
throw err;
}
};
/**
* Creates or Updates Zero Trust DNS rule for a given array of lists.
* @param {object[]} lists The lists to be used for the rule.
* @param {string} lists[].id The ID of the list.
* @param {string} lists[].name The name of the list.
* @param {string} listName The name of the list.
*/
export const upsertZeroTrustDNSRule = async (lists, listName) => {
// Create a Wirefilter expression to match DNS queries against all the lists
const wirefilterDNSExpression = lists
.filter(({ name }) => name.startsWith("CGPS List"))
.map(({ id }) => `any(dns.domains[*] in \$${id})`)
.join(" or ");
console.log("Checking DNS rule...");
await upsertZeroTrustRule(wirefilterDNSExpression, listName, ["dns"]);
};
/**
* Creates or Updates Zero Trust SNI rule for a given array of lists.
* @param {object[]} lists The lists to be used for the rule.
* @param {string} lists[].id The ID of the list.
* @param {string} lists[].name The name of the list.
* @param {string} listName The name of the list.
*/
export const upsertZeroTrustSNIRule = async (lists, listName) => {
// Create a Wirefilter expression to match SNI queries against all the lists
const wirefilterSNIExpression = lists
.filter(({ name }) => name.startsWith("CGPS List"))
.map(({ id }) => `any(net.sni.domains[*] in \$${id})`)
.join(" or ");
console.log("Creating SNI rule...");
await upsertZeroTrustRule(wirefilterSNIExpression, listName, ["l4"]);
};