Add ability to exclude indicators from calculation (#10)

* exclude indicators based on a predicate function
* version bump
* docs

Author: Tom Pearson

README.md

@@ -66,17 +66,43 @@ Take a look at the data source descriptions for a fuller description of the prop
 
 __indexMax__, optional. is the maximum value for the index score (minimum is always 0) by thefault this is 100, other typical values are 1 and 10.
 
-### indexCore.__indexedData__:Object
-An object where each property is an _entity_ in the index and that entity has properties for each of the properties in the indicators spreadsheet _and_ any calculated values.
-### indexCore.__indexStructure__:Object
-An object representing the structure of the index
-### indexCore.__adjustWeight(_indicatorID:String_, _weight:Number_)__
-A function that allows the user adjust the weighting of an indicator to give it a greater or lesser importance in theindex as whole. The index is recalculated after calling this function.
 ### indexCore.__adjustValue(_entityName:String_, _indicatorID:String_, _value:Number_)__
 A function that allows the user to adjust an entity's indicator score within the index the index is recalculated using the new value. The index is recalculated after calling this function.
 
 _NOTE: whislt the old value is retained there's currently no way to reset it._
+
+### indexCore.__adjustWeight(_indicatorID:String_, _weight:Number_)__
+A function that allows the user adjust the weighting of an indicator to give it a greater or lesser importance in theindex as whole. The index is recalculated after calling this function.
+
+### indexCore.__filterIndicators([_exclude:function_])__
+Allows the user to specify a predicate which excludes certain indicators from the calculation e.g.
+```js
+// if the indicator includes "b" in its  ID then ignore it
+indexCore.filterIndicators(indicator => String(indicator.id).indexOf('b')>0 )
+```
+
+You can clear the exclusion by calling the same function with no arguments
+```js
+// if the indicator includes "b" in its then id ignore it
+indexCore.filterIndicators()
+```
+
+### indexCore.__getEntity([indicatorId:String])__:Object
+
+### indexCore.__getEntities()__:Array
+
 ### indexCore.__getIndexMean([indicatorId:String], _[normalise:Boolean=false]_)__:Number
+
+### indexCore.__getIndicator()__:Object
+
+### indexCore.__getIndicatorLookup()__:Object
+
+### indexCore.__indexedData__:Object
+An object where each property is an _entity_ in the index and that entity has properties for each of the properties in the indicators spreadsheet _and_ any calculated values.
+
+### indexCore.__indexStructure__:Object
+An object representing the structure of the index
+r
 Returns the mean value for a given indicator accross all entities in the index which posses that value. Note that if an indicator does not posses that value it's excluded from the calculation and a sane result should be returned. Providing no _indicatorId_ will result in the mean of the top level index scores being returned. The second argument determines whether the result will be expressed as normalised (which is the default) or in the raw quantities of the indicator. i.e. if an indicator represents the number of hours of sunlight it could be better to express it un-normalised as in the mean number of hours rather than normalised to between e.g. 1-100.
 
 ---

index.js

@@ -1,27 +1,32 @@
+import { clone } from './src/utils.js';
 import {csvParse} from 'd3';
 import fs from 'fs';
 import indexCore from './src/index-core.js';
 
 const waterRootDir = 'data/wateroptimisation';
-const educationRootDir = 'data/education';
 
 const waterIndicators = csvParse(fs.readFileSync(`${waterRootDir}/indicators.csv`, 'utf-8'));
 const waterEntities = csvParse(fs.readFileSync(`${waterRootDir}/entities.csv`, 'utf-8'));
 
-const educationIndicators = csvParse(fs.readFileSync(`${educationRootDir}/indicators.csv`, 'utf-8'));
-const educationEntities = csvParse(fs.readFileSync(`${educationRootDir}/entities.csv`, 'utf-8'));
-
 const waterOptimisationIndex = indexCore(waterIndicators, waterEntities);
 
 
-const educationIndex = indexCore(educationIndicators, educationEntities);
-
 // console.log(waterOptimisationIndex.indexedData['Abu Dhabi']['1']);
 // console.log(waterOptimisationIndex.indexedData['Abu Dhabi'].value);
 // console.log(waterOptimisationIndex.indexStructure);
 // console.log(waterOptimisationIndex.getIndexMean('1.1'))
 // console.log(waterOptimisationIndex.getIndexMean('2.1.1'))
 // console.log(waterOptimisationIndex.getIndexMean())
 
-console.log(educationIndex.indexStructure);
-console.log(educationIndex.indexedData);
+// indicator 3.4.4 in the water index has .a and .b sub indicators 
+// for this indicator abu dhabi has different values for a and b
+const before = JSON.stringify(waterOptimisationIndex.getEntity('Abu Dhabi'), null, ' ')
+delete before.data;
+waterOptimisationIndex.filterIndicators(indicator=>{
+  return String(indicator.id).indexOf('b')>0; // if the indicator includes "b" in it's id ignore it
+})
+const after = JSON.stringify(waterOptimisationIndex.getEntity('Abu Dhabi'), null, ' ')
+delete after.data; // just for neater output (note data is the original data for an entity used to calculate the index)
+
+console.log('BEFORE', before);
+console.log('AFTER', after);

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@signal-noise/index-core",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "",
   "main": "src/index-core.js",
   "type": "module",

src/index-core.js

@@ -10,10 +10,10 @@ function indexCore(indicatorsData = [], entitiesData = [], indexMax = 100) {
   );
   const indexedData = {};
   let indexStructure = {};
+  let excludeIndicator = () => false; // by default no valid indicators are excluded
 
   function getEntity(entityName) {
     return indexedData[entityName];
-    //   return entitiesData.find((d) => d.name === entityName);
   }
 
   function getEntities() {
@@ -74,13 +74,14 @@ function indexCore(indicatorsData = [], entitiesData = [], indexMax = 100) {
 
   function indexEntity(entity, calculationList) {
     const newEntity = clone(entity);
-
     calculationList.forEach((indicatorID) => {
       if (newEntity[indicatorID]) { console.log('overwriting', indicatorID); }
-      // get the required component indicators tocalculate the parent value
+      // get the required component indicators to calculate the parent value
+      // this is a bit brittle maybe?
       const componentIndicators = indicatorsData
         .filter((indicator) => (indicator.id.indexOf(indicatorID) === 0
           && indicator.id.length === indicatorID.length + 2))
+        .filter((indicator) => !excludeIndicator(indicator))
         .map((indicator) => formatIndicator(indicator, newEntity, indexMax));
       // calculate the weighted mean of the component indicators on the newEntity
       // assign that value to the newEntity
@@ -103,6 +104,8 @@ function indexCore(indicatorsData = [], entitiesData = [], indexMax = 100) {
     }
     if (!e.user) e.user = {};
     e.user[indicatorID] = value;
+    //note the re-indexed value for the entity is not stored
+    //only the adjustment the re-indexed value is returned to the caller
     return indexEntity(Object.assign(clone(e), e.user));
   }
 
@@ -133,12 +136,17 @@ function indexCore(indicatorsData = [], entitiesData = [], indexMax = 100) {
     return tree;
   }
 
-  function calculateIndex(exclude = () => false) {
-    const onlyIdIndicators = indicatorsData.filter((i) => i.id.match(indicatorIdTest));
+  function calculateIndex() {
+    const onlyIdIndicators = indicatorsData
+      .filter((i) =>{
+        const isIndicator = i.id.match(indicatorIdTest)
+        const isExcluded = excludeIndicator(i);
+        return isIndicator && !isExcluded;
+      });
     // get a list of the values we need to calculate
     // in order of deepest in the heirachy to to shallowist
     const calculationList = onlyIdIndicators
-      .filter((i) => i.type === 'calculated' && !exclude(i))
+      .filter((i) => (i.type === 'calculated' && !excludeIndicator(i)))
       .map((i) => i.id)
       .sort((i1, i2) => (i2.split('.').length - i1.split('.').length));
 
@@ -147,6 +155,7 @@ function indexCore(indicatorsData = [], entitiesData = [], indexMax = 100) {
     entitiesData.forEach((entity) => {
       const indexedEntity = indexEntity(entity, calculationList);
       indexedEntity.data = entity;
+      indexedEntity.ts = new Date();
       indexedData[entity.name] = indexedEntity;
     });
   }
@@ -158,18 +167,24 @@ function indexCore(indicatorsData = [], entitiesData = [], indexMax = 100) {
     calculateIndex();
   }
 
+  function filterIndicators(exclude = ()=>false){
+    excludeIndicator = exclude;
+    calculateIndex();  
+  }
+
   calculateIndex();
 
   return {
     adjustValue,
     adjustWeight,
-    indexedData,
-    indexStructure,
-    getIndexMean,
+    filterIndicators,
     getEntity,
+    getEntities,
+    getIndexMean,
     getIndicator,
     getIndicatorLookup,
-    getEntities,
+    indexedData,
+    indexStructure,
   };
 }
 

test/index-core.test.js

@@ -20,7 +20,20 @@ test('getIndicatorMean index-core', ()=>{
   expect(waterOptimisationIndex.getIndexMean('1.2').toFixed(1)).toBe('87.9');
   expect(waterOptimisationIndex.getIndexMean('2.2').toFixed(1)).toBe('74.9');
   expect(waterOptimisationIndex.getIndexMean('2.1.1').toFixed(1)).toBe('89.8');
+  expect(waterOptimisationIndex.getIndexMean('3.1.1').toFixed(1)).toBe('38.2');
   expect(waterOptimisationIndex.getIndexMean('2').toFixed(1)).toBe('74.7');
   expect(waterOptimisationIndex.getIndexMean().toFixed(1)).toBe('69.5');
-  
+});
+
+test('filterIndicators index-core', ()=>{
+  const waterOptimisationIndex = indexCore(waterIndicators, waterEntities);
+  const excluder = (indicator) => (String(indicator.id).indexOf('b')>0); // if the indicator includes "b" in it's id ignore it
+  waterOptimisationIndex.filterIndicators(excluder)
+  expect(waterOptimisationIndex.getIndexMean('1').toFixed(1)).toBe('71.2');
+  expect(waterOptimisationIndex.getIndexMean('1.2').toFixed(1)).toBe('87.9');
+  expect(waterOptimisationIndex.getIndexMean('2.2').toFixed(1)).toBe('74.9');
+  expect(waterOptimisationIndex.getIndexMean('2.1.1').toFixed(1)).toBe('89.8');
+  expect(waterOptimisationIndex.getIndexMean('3.1.1').toFixed(1)).toBe('25.0');
+  expect(waterOptimisationIndex.getIndexMean('2').toFixed(1)).toBe('74.7');
+  expect(waterOptimisationIndex.getIndexMean().toFixed(1)).toBe('68.7');
 });