___INFO___

{
  "type": "MACRO",
  "id": "cvt_temp_public_id",
  "version": 1,
  "securityGroups": [],
  "displayName": "URL Cleaner",
  "categories": [
    "UTILITY"
  ],
  "description": "parses URLs and keeps only whitelisted or removes blacklisted parameters. Returns new full url, path or redacted query string only (optionally transformed to lower case)",
  "containerContexts": [
    "WEB"
  ]
}


___TEMPLATE_PARAMETERS___

[
  {
    "type": "TEXT",
    "name": "fullUrl",
    "displayName": "Full URL",
    "simpleValueType": true,
    "valueValidators": [
      {
        "type": "NON_EMPTY"
      }
    ],
    "alwaysInSummary": true
  },
  {
    "type": "SELECT",
    "name": "listMethod",
    "displayName": "List Method",
    "macrosInSelect": false,
    "selectItems": [
      {
        "value": "whitelist",
        "displayValue": "Whitelist"
      },
      {
        "value": "blacklist",
        "displayValue": "Blacklist"
      }
    ],
    "simpleValueType": true,
    "defaultValue": "whitelist",
    "help": "Switch between parameter whitelisting (all parameters are removed if not listed) or blacklisting (only listed parameters are removed).",
    "valueValidators": [
      {
        "type": "NON_EMPTY"
      }
    ],
    "alwaysInSummary": true
  },
  {
    "type": "CHECKBOX",
    "name": "useRegex",
    "checkboxText": "Allow Partial Match And RegEx",
    "simpleValueType": true,
    "help": "Check to use partial match (\"utm_\" would catch \"utm_medium\" and \"utm_source\" as well as \"autm_tk\") or regular expressions in your list.\n\nIf not active, parameters must match a list entry (not case-sensitive)."
  },
  {
    "type": "GROUP",
    "name": "grpWhitelist",
    "displayName": "Parameter Whitelist",
    "groupStyle": "NO_ZIPPY",
    "subParams": [
      {
        "type": "LABEL",
        "name": "infoWhitelist",
        "displayName": "Add all desired parameter names as separate rows. All other parameters will be removed."
      },
      {
        "type": "SIMPLE_TABLE",
        "name": "whitelistParams",
        "displayName": "Parameter Whitelist Table",
        "simpleTableColumns": [
          {
            "defaultValue": "",
            "displayName": "",
            "name": "paramName",
            "type": "TEXT"
          },
          {
            "defaultValue": "none",
            "displayName": "Value Processing",
            "name": "valueFunction",
            "type": "SELECT",
            "selectItems": [
              {
                "value": "none",
                "displayValue": "none"
              },
              {
                "value": "lower",
                "displayValue": "lower"
              }
            ]
          }
        ],
        "help": ""
      }
    ],
    "enablingConditions": [
      {
        "paramName": "listMethod",
        "paramValue": "whitelist",
        "type": "EQUALS"
      }
    ]
  },
  {
    "type": "GROUP",
    "name": "grpBlacklist",
    "displayName": "Parameter Blacklist",
    "groupStyle": "NO_ZIPPY",
    "subParams": [
      {
        "type": "LABEL",
        "name": "infoBlacklist",
        "displayName": "Add all parameter names that should be removed from the URL as separate rows. All other parameters will be preserved."
      },
      {
        "type": "SIMPLE_TABLE",
        "name": "blacklistParams",
        "displayName": "Parameter Blacklist Table",
        "simpleTableColumns": [
          {
            "defaultValue": "",
            "displayName": "",
            "name": "paramName",
            "type": "TEXT"
          }
        ],
        "help": ""
      }
    ],
    "enablingConditions": [
      {
        "paramName": "listMethod",
        "paramValue": "blacklist",
        "type": "EQUALS"
      }
    ]
  },
  {
    "type": "CHECKBOX",
    "name": "lowercaseHostname",
    "checkboxText": "Hostname To Lowercase",
    "simpleValueType": true,
    "defaultValue": true
  },
  {
    "type": "CHECKBOX",
    "name": "lowercasePathname",
    "checkboxText": "Pathname To Lowercase",
    "simpleValueType": true,
    "defaultValue": true
  },
  {
    "type": "CHECKBOX",
    "name": "lowercaseSearchParamsKeys",
    "checkboxText": "All SearchParams Keys To Lowercase",
    "simpleValueType": true,
    "help": "Check this option to change the result to lowercase characters. Note: modification happens before whitelisting and application of RegEx filters. This means that both functions will be case-insensitive.",
    "defaultValue": true
  },
  {
    "type": "CHECKBOX",
    "name": "lowercaseSearchParamsValues",
    "checkboxText": "All SearchParams Values To Lowercase",
    "simpleValueType": true,
    "help": "Check this option to change the result to lowercase characters. Note: modification happens before whitelisting and application of RegEx filters. This means that both functions will be case-insensitive.",
    "defaultValue": false
  },
  {
    "type": "CHECKBOX",
    "name": "redactValues",
    "checkboxText": "Redact Values",
    "simpleValueType": true,
    "alwaysInSummary": false,
    "help": "Optional redaction of remaining parameter values with regex patterns."
  },
  {
    "type": "GROUP",
    "name": "grpRegex",
    "displayName": "Regex List",
    "groupStyle": "NO_ZIPPY",
    "subParams": [
      {
        "type": "LABEL",
        "name": "infoRegex",
        "displayName": "Add one or multiple rows with regex expressions to apply to all remaining parameter values. Matching strings will be replaced with [REDACTED]."
      },
      {
        "type": "SIMPLE_TABLE",
        "name": "redactPatterns",
        "displayName": "Regex Pattern",
        "simpleTableColumns": [
          {
            "defaultValue": "",
            "displayName": "",
            "name": "rgx",
            "type": "TEXT"
          }
        ],
        "help": "Example for email addresses: [a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+"
      },
      {
        "type": "TEXT",
        "name": "redactReplacement",
        "displayName": "Replacement String",
        "simpleValueType": true,
        "defaultValue": "[REDACTED]",
        "valueValidators": [
          {
            "type": "NON_EMPTY"
          }
        ],
        "help": "Enter string to replace all matches with regex expressions."
      }
    ],
    "enablingConditions": [
      {
        "paramName": "redactValues",
        "paramValue": true,
        "type": "EQUALS"
      }
    ]
  },
  {
    "type": "SELECT",
    "name": "resultFormat",
    "displayName": "Result Format",
    "macrosInSelect": false,
    "selectItems": [
      {
        "value": "paramsOnly",
        "displayValue": "Clean Parameter String"
      },
      {
        "value": "pageOnly",
        "displayValue": "Clean Page Path With Parameters"
      },
      {
        "value": "cleanUrl",
        "displayValue": "Clean URL"
      }
    ],
    "simpleValueType": true,
    "defaultValue": "cleanUrl",
    "valueValidators": [
      {
        "type": "NON_EMPTY"
      }
    ],
    "alwaysInSummary": true
  },
  {
    "type": "TEXT",
    "name": "maxLen",
    "displayName": "Maximum URL Length",
    "simpleValueType": true,
    "valueValidators": [
      {
        "type": "NON_EMPTY"
      },
      {
        "type": "POSITIVE_NUMBER"
      }
    ],
    "defaultValue": 420
  },
  {
    "type": "CHECKBOX",
    "name": "httpsConvert",
    "checkboxText": "Convert to HTTPS",
    "simpleValueType": true,
    "defaultValue": true
  }
]


___SANDBOXED_JS_FOR_WEB_TEMPLATE___

var Object = require("Object");
var parseUrl = require("parseUrl");
var getType = require("getType");
var log = require("logToConsole");
log("GTM msg data:" + data);

const cleanerFuncs = {
    none: (val) => val,
    lower: (val) => val.toLowerCase()
};

const compare = (a, b) => {
  if (a.priority < b.priority) {
    return -1;
  }
  if (a.priority > b.priority) {
    return 1;
  }
  return 0;
};

const joinMax = (cleanParams, maxLen) => {
  let result = "";
  const otherPriority = 100;

  const paramsPriority = {
    utm_id: 0,
    gclid: 0,
    gbraid: 0,
    wbraid: 0,
    utm_source: 1,
    utm_medium: 1,
    utm_campaign: 1,
    utm_term: 1,
    utm_content: 1,
    utm_query: 2,
    utm_marketing_tactic: 2,
    utm_creative_format: 2,
    utm_source_platform: 2,
    fbclid: 2,
    twclid: 2,
    ttclid: 2,
    li_fat_id: 2,
    ScClid: 2,
    msclkid: 2,
  };

  let cleanParamsWithPriorities = [];
  for (let index = 0; index < cleanParams.length; index++) {
    const param = cleanParams[index];
    const key = param.split("=")[0];
    const priority =
      Object.keys(paramsPriority).indexOf(key) > -1 ? paramsPriority[key] : otherPriority;
    cleanParamsWithPriorities.push({
      key: key,
      value: param,
      priority: priority,
    });
  }

  cleanParamsWithPriorities = cleanParamsWithPriorities.sort(compare);

  for (let index = 0; index < cleanParamsWithPriorities.length; index++) {
    const param = cleanParamsWithPriorities[index].value;
    const resultLength = result.length;
    const paramLenth = param.length;
    if (maxLen - (resultLength + paramLenth) > 0) {
      result += param + "&";
    } else break;
  }
  return result.slice(0, -1);
};

const lowerCaseByConfig = (value, toLower) => {
    return toLower ? value.toLowerCase() : value;
};

var lm = data.listMethod || "whitelist";
log("GTM msg lm:" + lm);

var testParams = (lm==="whitelist") ? data.whitelistParams : data.blacklistParams;
var inUrl = parseUrl(data.fullUrl);
const sp = inUrl.searchParams;

if  (getType(inUrl.hash)!=="undefined" ) {
    const hashUrl = "https://test.com?" + inUrl.hash.slice(1); // PP said check this line
    const hashUrlParsed = parseUrl(hashUrl);
    if ( getType(hashUrlParsed.searchParams)!=="undefined" ) {
        Object.keys(hashUrlParsed.searchParams).forEach((key) => {
            sp[key] = hashUrlParsed.searchParams[key];
        });
    }
}

var cleanParams = [];
for ( var prm of Object.entries(sp) ) {
    var k = prm[0] || "";
    var vl = (prm.length>1) ? prm[1] : "";
    if (vl==="") continue;
    vl = lowerCaseByConfig(vl, data.lowercaseSearchParamsValues);
    k = lowerCaseByConfig(k, data.lowercaseSearchParamsKeys);
    let keepParam = (lm==="whitelist") ? false : true;
    for (let index = 0; index < testParams.length; index++) {
        const testParam = testParams[index].paramName;
        keepParam = (data.useRegex) ? getType(k.match(testParam))!=="null" : testParam===k.toLowerCase();
        keepParam = (lm==="whitelist") ? keepParam : !keepParam;
        if (keepParam && lm==="whitelist") {
            const valueFunctionName = testParams[index].valueFunction;
            const cleanerFunc = cleanerFuncs[valueFunctionName];
            vl = cleanerFunc(vl);
            break;
        } else if (lm!=="whitelist" && !keepParam) {
            break;
        }
    }

    if (keepParam===true) {
        if ( data.redactValues===true && data.redactPatterns && data.redactPatterns.length>0 ) {
            data.redactPatterns.forEach((pat) => {
                const redactInfo = vl.match(pat.rgx);
                if (redactInfo) {
                   vl = vl.replace(redactInfo, data.redactReplacement);
                } else if ( 
                  pat.rgx.substring(0,2)==="%%" && 
                  pat.rgx.substring(pat.rgx.length-2)==="%%" && 
                  "%%" + k.toLowerCase() + "%%"===pat.rgx 
                ) {
                    vl = data.redactReplacement;
                }
            });
        }
        cleanParams.push(k +"="+ vl);
    }
}

let maxLen = data.maxLen;
var hst = inUrl.hostname;
var pth = inUrl.pathname;
var protocol = data.httpsConvert ? "https:" : inUrl.protocol;

if (data.resultFormat==="pageOnly") {
    maxLen = maxLen - pth.length;
}
if (data.resultFormat==="pageOnly") {
   // Do Nothing
} else {
    maxLen = maxLen - (protocol +"//"+ hst + pth).length; // PP said check ":" returned
}
log("GTM msg maxLen:" + maxLen);

// var cleanQuery = cleanParams.join("&");
var cleanQuery = joinMax(cleanParams, maxLen);
if (cleanQuery.length>0) cleanQuery = "?" + cleanQuery;

if (data.lowercaseHostname) hst = hst.toLowerCase();
if (data.lowercasePathname) pth = pth.toLowerCase();
//if (data.lowercaseSearchParams) cleanQuery = cleanQuery.toLowerCase();

if (data.resultFormat==="paramsOnly") return cleanQuery;
if (data.resultFormat==="pageOnly") return pth + cleanQuery;
return protocol + "//" + hst + pth + cleanQuery; // PP said check ":" returned


___WEB_PERMISSIONS___

[
  {
    "instance": {
      "key": {
        "publicId": "logging",
        "versionId": "1"
      },
      "param": [
        {
          "key": "environments",
          "value": {
            "type": 1,
            "string": "debug"
          }
        }
      ]
    },
    "clientAnnotations": {
      "isEditedByUser": true
    },
    "isRequired": true
  }
]


___TESTS___

scenarios:
- name: Whitelist, Full URL, lowercase
  code: |-
    mockData.whitelistParams = [{paramName: "utm_source","valueFunction":"none"}, {paramName: "utm_medium","valueFunction":"none"}];

    let variableResult = runCode(mockData);
    assertThat(variableResult).isEqualTo("https://www.example.com/page.html?utm_medium=test&utm_source=check");
- name: Blacklist, Page Only
  code: |-
    mockData.blacklistParams = [{paramName: "foo"}, {paramName: "fbclid"}];
    mockData.listMethod = "blacklist";
    mockData.resultFormat = "pageOnly";

    let variableResult = runCode(mockData);
    assertThat(variableResult).isEqualTo("/page.html?utm_medium=test&utm_source=check&RANDOM=email@example.com");
- name: Blacklist, Params Only, Redacted Email
  code: |-
    mockData.blacklistParams = [{paramName: "foo","valueFunction":"none"}, {paramName: "fbclid","valueFunction":"none"}];
    //sorry... but more complex patterns break tests (but work in preview and live)
    const rp = [{rgx:'email@example.com'}];
    mockData.redactValues = true;
    mockData.redactPatterns = rp;

    mockData.lowercaseSearchParamsKeys = true;
    mockData.listMethod = "blacklist";
    mockData.resultFormat = "paramsOnly";

    let variableResult = runCode(mockData);
    assertThat(variableResult).isEqualTo("?utm_medium=test&utm_source=check&random=[gone]");
- name: Whitelist, Regex List Items
  code: |-
    mockData.whitelistParams = [{paramName: "utm_","valueFunction":"none"}, {paramName: "fo+","valueFunction":"none"}];
    mockData.useRegex = true;

    let variableResult = runCode(mockData);
    assertThat(variableResult).isEqualTo("https://www.example.com/page.html?utm_medium=test&utm_source=check&foo=bar");
- name: Blacklist, Regex List Items
  code: |-
    mockData.fullUrl = "https://www.example.com/page.html?utm_medium=test&utm_source=check&autm_tk=check2&fbclid=something&foo=bar";
    mockData.blacklistParams = [{paramName: "utm_"}, {paramName: "fop"}];
    mockData.listMethod = "blacklist";
    mockData.useRegex = true;

    let variableResult = runCode(mockData);
    assertThat(variableResult).isEqualTo("https://www.example.com/page.html?fbclid=something&foo=bar");
- name: Blacklist, Better Regex
  code: |-
    mockData.fullUrl = "https://www.example.com/page.html?utm_medium=test&utm_source=check&autm_tk=check2&fbclid=something&foo=bar";
    mockData.blacklistParams = [{paramName: "^utm_.+"},];
    mockData.listMethod = "blacklist";
    mockData.useRegex = true;

    let variableResult = runCode(mockData);
    assertThat(variableResult).isEqualTo("https://www.example.com/page.html?autm_tk=check2&fbclid=something&foo=bar");
- name: Special Functions - Use Param Name to Redact Value
  code: |-
    mockData.listMethod = "blacklist";
    mockData.blacklistParams = [];
    mockData.fullUrl = "https://WWW.example.com/page.html?utm_medium=test&utm_source=check&foo=bar&test1=stay1&test2=stay2&test3=stay3";
    mockData.lowercaseUrl = true;
    mockData.useRegex = true;
    const rp = [{rgx:'%%foo%%'}];
    mockData.redactValues = true;
    mockData.redactPatterns = rp;

    let variableResult = runCode(mockData);
    assertThat(variableResult).isEqualTo("https://www.example.com/page.html?utm_medium=test&utm_source=check&foo=[gone]&test1=stay1&test2=stay2&test3=stay3");
- name: Whitelist, Full URL with hash, lowercase
  code: |-
    mockData.fullUrl = "https://WWW.Example.com/Page.html?Gclid=Test&utm_source=check&fbclid=something&foo=bar&RANDOM=email@example.com#hash_key=Hash_Value";

    mockData.whitelistParams = [{paramName: "utm_source","valueFunction":"lower"}, {paramName: "gclid","valueFunction":"none"},{paramName: "hash_key","valueFunction":"lower"}];
    mockData.lowercaseHostname = true;
    mockData.lowercasePathname = true;
    mockData.lowercaseSearchParamsKeys = true;
    mockData.lowercaseSearchParamsValues = false;


    let variableResult = runCode(mockData);
    assertThat(variableResult).isEqualTo("https://www.example.com/page.html?gclid=Test&utm_source=check&hash_key=hash_value");
- name: Whitelist, Full URL with hash and exclamation mark, lowercase
  code: |-
    mockData.fullUrl = "https://WWW.Example.com/Page.html?Gclid=Test&utm_source=check&fbclid=something&foo=bar&RANDOM=email@example.com#!";

    mockData.whitelistParams = [{paramName: "utm_source","valueFunction":"lower"}, {paramName: "gclid","valueFunction":"none"},{paramName: "hash_key","valueFunction":"lower"}];
    mockData.lowercaseHostname = true;
    mockData.lowercasePathname = true;
    mockData.lowercaseSearchParamsKeys = true;
    mockData.lowercaseSearchParamsValues = false;


    let variableResult = runCode(mockData);
    assertThat(variableResult).isEqualTo("https://www.example.com/page.html?gclid=Test&utm_source=check");
- name: Whitelist, Regex List Items, URL with hash
  code: |-
    mockData.fullUrl = "https://online.test.co.uk/Originations/launchpage.aspx?productType=RegularSaver&productCode=6074&utm_source=google&utm_medium=organic&utm_campaign=google_organic&utm_content=&gclid=123sDFgf4&utm_term=";


    mockData.whitelistParams = [{paramName: "utm_","valueFunction":"lower"}, {paramName: "product","valueFunction":"lower"}, {paramName: "gclid","valueFunction":"none"}];
    mockData.useRegex = true;

    mockData.lowercaseHostname = true;
    mockData.lowercasePathname = true;
    mockData.lowercaseSearchParamsKeys = true;
    mockData.lowercaseSearchParamsValues = false;


    let variableResult = runCode(mockData);
    assertThat(variableResult).isEqualTo("https://online.test.co.uk/originations/launchpage.aspx?producttype=regularsaver&productcode=6074&utm_source=google&utm_medium=organic&utm_campaign=google_organic&gclid=123sDFgf4");
- name: Whitelist lowercase pathname
  code: |-
    mockData.fullUrl = "https://online.test.co.uk/Originations/EIDConfirmationSuccess.aspx";

    mockData.whitelistParams = [{paramName: "utm_source","valueFunction":"lower"}, {paramName: "gclid","valueFunction":"none"},{paramName: "hash_key","valueFunction":"lower"}];
    mockData.lowercaseHostname = true;
    mockData.lowercasePathname = true;
    mockData.lowercaseSearchParamsKeys = true;
    mockData.lowercaseSearchParamsValues = false;


    let variableResult = runCode(mockData);
    assertThat(variableResult).isEqualTo("https://online.test.co.uk/originations/eidconfirmationsuccess.aspx");
- name: Length lower max length
  code: |-
    mockData.fullUrl = "https://d123456789.com/?utm_source=u1";

    mockData.whitelistParams = [{"paramName":"utm_source","valueFunction":"none"}];
    mockData.listMethod = "whitelist";


    let variableResult = runCode(mockData);
    assertThat(variableResult).isEqualTo("https://d123456789.com/?utm_source=u1");
- name: Length more than max length
  code: |-
    mockData.fullUrl = "https://d123456789.com/?utm_source=u1";

    mockData.whitelistParams = [{"paramName":"utm_source","valueFunction":"none"}];
    mockData.listMethod = "whitelist";
    mockData.maxLen = "30";

    let variableResult = runCode(mockData);
    assertThat(variableResult).isEqualTo("https://d123456789.com/");
- name: Length more than max length, only 1 param
  code: |-
    mockData.fullUrl = "https://d123456789.com/?utm_source=s1&utm_medium=m1";

    mockData.whitelistParams = [{"paramName":"utm_source","valueFunction":"none"},{"paramName":"utm_medium","valueFunction":"none"}];
    mockData.listMethod = "whitelist";
    mockData.maxLen = "41";

    let variableResult = runCode(mockData);
    assertThat(variableResult).isEqualTo("https://d123456789.com/?utm_source=s1");
- name: Convert to HTTPS
  code: |-
    mockData.fullUrl = "http://d123456789.com/?utm_source=u1";

    mockData.whitelistParams = [{"paramName":"utm_source","valueFunction":"none"}];
    mockData.listMethod = "whitelist";
    mockData.httpsConvert = true;

    let variableResult = runCode(mockData);
    assertThat(variableResult).isEqualTo("https://d123456789.com/?utm_source=u1");
- name: Chnage params order based on priority
  code: |-
    mockData.fullUrl = "https://d123456789.com/?test=1&utm_source=u1&gclid=g1";

    mockData.whitelistParams = [{"paramName":"utm_source","valueFunction":"none"},{"paramName":"gclid","valueFunction":"none"},{"paramName":"test","valueFunction":"none"},];
    mockData.listMethod = "whitelist";


    let variableResult = runCode(mockData);
    assertThat(variableResult).isEqualTo("https://d123456789.com/?gclid=g1&utm_source=u1&test=1");
- name: Delete params based on priority
  code: |-
    mockData.fullUrl = "https://d123456789.com/?test=1&utm_source=u1&gclid=g1";

    mockData.whitelistParams = [{"paramName":"utm_source","valueFunction":"none"},{"paramName":"gclid","valueFunction":"none"},{"paramName":"test","valueFunction":"none"},];
    mockData.listMethod = "whitelist";
    mockData.maxLen = "35";

    let variableResult = runCode(mockData);
    assertThat(variableResult).isEqualTo("https://d123456789.com/?gclid=g1");
setup: |-
  let mockData = {
    fullUrl: "https://WWW.example.com/page.html?utm_medium=test&utm_source=check&fbclid=something&foo=bar&RANDOM=email@example.com",
    listMethod: "whitelist",
    redactValues: false,
    redactReplacement: "[gone]",
    resultFormat: "cleanUrl",
    maxLen: "500"

  };


___NOTES___

Created on 4.5.2022, 21:23:46