{"steps":[{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":100,"type":"map","name":"Add metadata for active directory device data","source":[{"table":"ActiveDirectoryDevices*"}],"target":{"name":"__snapshot_ActiveDirectoryDevices"}},{"mapRemaining":false,"columns":{"import_data_source_id":{"value":"AD"},"source":{"value":"AD"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_ActiveDirectoryDevices] WHERE __source_id =' || __source_id"},"import_id":{"value":null},"device_key":{"sql":"!getFQDNWithCNDefault(dNSHostName,cn,distinguishedName)"},"device_name":{"column":"cn","transform":"Uppercase"},"hostname":{"column":"cn","transform":"Uppercase"},"department":{"column":"department"},"import_device_type_id":{"value":"AD_COMPUTEROBJECT"},"import_device_status_id":{"sql":"!GetStatusFromUacAndLogon(userAccountControl_resolved, lastLogonTimestamp_resolved)"},"fqdn":{"sql":"!getFQDN(dNSHostName,cn,distinguishedName)"},"import_domain_id":{"sql":"!getDomainFromDN(distinguishedName)"},"import_operating_system_id":{"sql":"!getCombinedOperatingSystemString(operatingSystem, operatingSystemVersion, operatingSystemServicePack)"},"last_logon_timestamp":{"sql":"date(lastLogonTimestamp_resolved)"},"ip_address":{"value":null},"import_org_level_2_id":{"value":"unknown"}},"id":101,"type":"map","name":"Transform active directory device data","source":[{"step":100}]},{"mapRemaining":true,"columns":{},"id":197,"type":"map","name":"Combine directory service device data","source":[{"step":101}]},{"conditions":[{"$sql":"[device_key] IS NOT NULL"}],"top":0,"id":198,"type":"filter","name":"Filter out directory service devices with missing information","source":{"step":197}},{"by":["device_key"],"strategy":{"type":"maxvalue","column":"last_logon_timestamp"},"id":199,"type":"deduplicate","name":"Find most recent logon time for each directory service device","source":{"step":198}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":200,"type":"map","name":"Add metadata for SCCM device data (SQL)","source":[{"table":"SCCMSQLDevices*"}],"target":{"name":"__snapshot_SCCMSQLDevices"}},{"mapRemaining":false,"columns":{"import_data_source_id":{"value":"SCCM_SQL"},"source":{"value":"SCCM_SQL"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_SCCMSQLDevices] WHERE __source_id =' || __source_id"},"import_id":{"value":null},"device_key":{"column":"device_key"},"import_org_level_2_id":{"sql":"!changeNullStringToUnknown(import_org_level_2_id)"},"device_name":{"column":"device_name"},"device_manufacturer":{"column":"device_manufacturer"},"device_model":{"column":"device_model"},"import_user_id":{"column":"import_user_id"},"import_device_type_id":{"sql":"!SCCM_Device_Type_ID_Mapping(import_device_type_id)"},"import_device_status_id":{"column":"import_device_status_id"},"ip_address":{"column":"ip_address"},"mac_address":{"column":"mac_address"},"inventory_number":{"column":"inventory_number","compact":true},"serial_number":{"column":"serial_number"},"installation_date":{"sql":"!transformDate(installation_date)"},"inventory_date":{"sql":"!transformDate(inventory_date)"},"fqdn":{"column":"fqdn"},"import_cpu_type_id":{"column":"import_cpu_type_id"},"cpu_speed":{"column":"cpu_speed"},"ram":{"column":"ram"},"storage":{"column":"storage"},"graphics":{"column":"graphics"},"network":{"column":"network"},"bios":{"column":"bios"},"import_operating_system_id":{"column":"operating_system"},"cpu_core_count":{"column":"cpu_core_count"},"cpu_chip_count":{"column":"cpu_chip_count"},"import_domain_id":{"column":"import_domain_id"}},"id":201,"type":"map","name":"Transform SCCM device data (SQL)","source":[{"step":200}]},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":202,"type":"map","name":"Add metadata for RayVentory device data","source":[{"table":"InventoryDevice"}],"target":{"name":"__snapshot_InventoryDevice"}},{"mapRemaining":false,"columns":{"import_data_source_id":{"value":"RV"},"source":{"value":"RV"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_InventoryDevice] WHERE __source_id =' || __source_id"},"import_id":{"value":null},"device_key":{"column":"device_key"},"import_org_level_2_id":{"sql":"!changeNullStringToUnknown(import_org_level_2_id)"},"device_name":{"column":"device_name"},"device_manufacturer":{"column":"device_manufacturer"},"device_model":{"column":"device_model"},"import_user_id":{"column":"import_user_id"},"import_device_type_id":{"column":"import_device_type_id"},"import_device_status_id":{"sql":"Lower(import_device_status_id)"},"ip_address":{"column":"ip_address"},"mac_address":{"column":"mac_address"},"inventory_number":{"column":"inventory_number","compact":true},"serial_number":{"column":"serial_number"},"installation_date":{"sql":"!transformDate(installation_date)"},"inventory_date":{"sql":"!transformDate(inventory_date)"},"fqdn":{"sql":"!checkFQDN(fqdn)"},"import_cpu_type_id":{"column":"import_cpu_type_id"},"cpu_speed":{"column":"cpu_speed"},"ram":{"column":"ram"},"storage":{"column":"storage"},"graphics":{"column":"graphics"},"network":{"column":"network"},"bios":{"column":"bios"},"import_operating_system_id":{"column":"import_operating_system_id"},"cpu_core_count":{"column":"cpu_core_count"},"cpu_chip_count":{"column":"cpu_chip_count"},"import_domain_id":{"column":"import_domain_id"},"hyper_threading_act":{"column":"hyper_threading_act"},"cpu_thread_count":{"column":"cpu_thread_count"},"limit_cpu_core_count":{"column":"limit_cpu_core_count"},"cpu_core_count_limited":{"column":"cpu_core_count_limited"},"uuid":{"column":"uuid","compact":true}},"id":203,"type":"map","name":"Transform RayVentory device data","source":[{"step":202}]},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":204,"type":"map","name":"Add metadata for SCCM device data (WMI)","source":[{"table":"SCCM_WMI*-Hardware"}],"target":{"name":"__snapshot_SCCM_WMI_Hardware"}},{"mapRemaining":false,"columns":{"import_data_source_id":{"value":"SCCM_WMI"},"source":{"value":"SCCM_WMI"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_SCCM_WMI_Hardware] WHERE __source_id =' || __source_id"},"import_id":{"value":null},"device_key":{"column":"device_key"},"device_name":{"column":"device_name"},"device_manufacturer":{"column":"device_manufacturer"},"device_model":{"column":"device_model"},"import_device_type_id":{"column":"import_device_type_id"},"import_device_status_id":{"sql":"CASE WHEN active = 1 THEN 'active' ELSE 'inactive' END"},"ip_address":{"column":"ip_address"},"mac_address":{"column":"mac_address"},"serial_number":{"column":"serial_number"},"installation_date":{"sql":"!transformDate(installation_date)"},"inventory_date":{"sql":"!transformDate(inventory_date)"},"fqdn":{"column":"fqdn"},"import_cpu_type_id":{"column":"import_cpu_type_id"},"cpu_speed":{"column":"cpu_speed"},"ram":{"column":"ram"},"storage":{"column":"storage"},"graphics":{"column":"graphics"},"network":{"column":"network"},"bios":{"column":"bios"},"import_operating_system_id":{"column":"import_operating_system_id"},"cpu_core_count":{"column":"cpu_core_count"},"cpu_chip_count":{"column":"cpu_chip_count"},"import_domain_id":{"column":"import_domain_id"},"import_org_level_2_id":{"value":"unknown"}},"id":205,"type":"map","name":"Transform SCCM device data (WMI)","source":[{"step":204}]},{"mapRemaining":true,"columns":{},"id":297,"type":"map","name":"Combine software deployment and discovery device data","source":[{"step":201},{"step":203},{"step":205}]},{"conditions":[{"$sql":"[device_key] IS NOT NULL"}],"top":0,"id":298,"type":"filter","name":"Filter out software deployment and discovery devices with missing information","source":{"step":297}},{"by":["device_key"],"strategy":{"type":"maxvalue","column":"inventory_date"},"id":299,"type":"deduplicate","name":"Find most recent inventory date for each software deployment and discovery device","source":{"step":298}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":300,"type":"map","name":"Add metadata for vCenter host device data","source":[{"table":"vcenter*-vmhosts"}],"target":{"name":"__snapshot_vcenter_vmhosts"}},{"mapRemaining":false,"columns":{"import_data_source_id":{"value":"vCenter"},"source":{"value":"vCenter"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_vcenter_vmhosts] WHERE __source_id =' || __source_id"},"device_key":{"column":"name"},"device_name":{"sql":"!getNameFromFQDN(name)"},"fqdn":{"sql":"!checkFQDN(name)"},"import_domain_id":{"sql":"!getDomainFromFQDN(name)"},"import_device_status_id":{"sql":"!GetDeviceStatusFromState(powerState)"},"import_operating_system_id":{"value":null},"ip_address":{"value":null},"mac_address":{"value":null},"device_manufacturer":{"column":"manufacturer"},"device_model":{"column":"model"},"import_device_type_id":{"value":"vmware_esx_host"},"cpu_chip_count":{"column":"numCPUs"},"cpu_core_count":{"column":"numCpuCoresOverall"},"cpu_speed":{"column":"totalHost_Mhz"},"import_cpu_type_id":{"column":"processorType"},"ram":{"value":null},"storage":{"value":null},"bios":{"value":null},"import_org_level_2_id":{"value":"unknown"}},"id":301,"type":"map","name":"Transform vCenter host device data","source":[{"step":300}]},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":302,"type":"map","name":"Add metadata for vCenter VM device data","source":[{"table":"vcenter*-vms"}],"target":{"name":"__snapshot_vcenter_vms"}},{"mapRemaining":false,"columns":{"import_data_source_id":{"value":"vCenter"},"source":{"value":"vCenter"},"device_key":{"sql":"!GetKeyFromFqdnAndName(fqdn, name)"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_vcenter_vms] WHERE __source_id =' || __source_id"},"device_name":{"sql":"!getNameFromFQDNORName(fqdn, name)"},"import_domain_id":{"sql":"!getDomainFromFQDNORNAME(fqdn, name)"},"fqdn":{"sql":"!checkFQDNORNAME(fqdn, name)"},"import_device_type_id":{"value":"vmware_virtual_machine"},"ip_address":{"column":"ipaddress","compact":true},"mac_address":{"column":"macaddress","compact":true},"import_device_status_id":{"sql":"!GetDeviceStatusFromState(powerState)"},"import_operating_system_id":{"sql":"!getOperatingSystem(installedOS, settingOS)"},"cpu_speed":{"value":"EMPTY"},"import_cpu_type_id":{"value":"EMPTY"},"cpu_chip_count":{"value":"EMPTY"},"cpu_core_count":{"column":"cpuCoreCount"},"ram":{"column":"ram"},"storage":{"column":"totalHDD_GB"},"bios":{"column":"firmware"},"uuid":{"column":"uuid","compact":true},"import_org_level_2_id":{"value":"unknown"}},"id":303,"type":"map","name":"Transform vCenter VM device data","source":[{"step":302}]},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":304,"type":"map","name":"Add metadata for vCenter cluster device data","source":[{"table":"vcenter*-clusters"}],"target":{"name":"__snapshot_vcenter_clusters"}},{"mapRemaining":false,"columns":{"import_data_source_id":{"value":"vCenter"},"source":{"value":"vCenter"},"import_id":{"value":null},"device_key":{"column":"name"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_vcenter_clusters] WHERE __source_id =' || __source_id"},"device_name":{"column":"name"},"import_device_type_id":{"value":"vmware_esx_cluster"},"import_device_status_id":{"value":"active"},"device_manufacturer":{"value":null},"device_model":{"value":null},"import_operating_system_id":{"value":null},"import_domain_id":{"value":null},"ip_address":{"value":null},"mac_address":{"value":null},"fqdn":{"value":null},"cpu_chip_count":{"value":null},"cpu_core_count":{"value":null},"cpu_speed":{"value":null},"import_cpu_type_id":{"value":null},"ram":{"value":null},"storage":{"value":null},"bios":{"value":null},"import_org_level_2_id":{"value":"unknown"}},"id":305,"type":"map","name":"Transform vCenter cluster device data","source":[{"step":304}]},{"mapRemaining":true,"columns":{},"id":397,"type":"map","name":"Combine hypervisor device data","source":[{"step":301},{"step":303},{"step":305}]},{"conditions":[{"$sql":"[device_key] IS NOT NULL AND [import_device_status_id] = 'active'"}],"top":0,"id":398,"type":"filter","name":"Filter out inactive hypervisor device and hypervisor device with missing information","source":{"step":397}},{"by":["device_key"],"strategy":{"type":"any"},"id":399,"type":"deduplicate","name":"Deduplicate hypervisor device data on device key","source":{"step":398}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":400,"type":"map","name":"Add metadata for AWS RDS device data","source":[{"table":"aws_rds*-instances"}],"target":{"name":"__snapshot_aws_rds_instances"}},{"mapRemaining":false,"columns":{"import_data_source_id":{"value":"aws_rds"},"source":{"value":"aws_rds"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_aws_rds_instances] WHERE __source_id =' || __source_id"},"import_device_type_id":{"sql":"'AWS_' || instanceType"},"import_device_status_id":{"value":"active"},"device_key":{"sql":"accountId || ' ' || instanceId"},"device_name":{"column":"instanceId"},"import_org_level_2_id":{"value":"unknown"}},"id":401,"type":"map","name":"Transform AWS RDS device data","source":[{"step":400}]},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":402,"type":"map","name":"Add metadata for AWS EC2 device data","source":[{"table":"aws_ec2*-instances"}],"target":{"name":"__snapshot_aws_ec2_instances"}},{"mapRemaining":false,"columns":{"import_data_source_id":{"value":"aws_ec2_instances"},"source":{"value":"aws_ec2_instances"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_aws_ec2_instances] WHERE __source_id =' || __source_id"},"import_device_type_id":{"sql":"'EC2_' || instanceType"},"import_device_status_id":{"sql":"!GetDeviceStatusFromState(state)"},"instanceId":{"column":"instanceId"},"inventory_number":{"sql":"('AWS EC2 instance id: ' || instanceId)"},"cpu_core_count":{"column":"cores"},"ip_address":{"sql":"COALESCE(publicipaddress, privateIpAddress)"},"import_org_level_2_id":{"value":"unknown"}},"id":403,"type":"map","name":"Transform AWS EC2 device data","source":[{"step":402}]},{"mapRemaining":false,"columns":{"instanceId":{"column":"instanceId"},"aws_ec2_tags_key":{"column":"key"},"aws_ec2_tags_value":{"column":"value"}},"id":404,"type":"map","name":"Query for AWS EC2 tags","source":[{"table":"aws_ec2*-tags"}],"target":{"name":"__snapshot_aws_ec2_tags"}},{"conditions":[{"$sql":"[aws_ec2_tags_key] = 'Name'"}],"top":0,"id":405,"type":"filter","name":"Filter out AWS EC2 tags not describing names","source":{"step":404}},{"strategy":"left","sources":[{"step":403},{"step":405}],"on":["instanceId"],"conflicts":{},"id":406,"type":"join","name":"Enrich AWS EC2 device data with tags"},{"mapRemaining":false,"columns":{"source":{"column":"source"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"column":"__source_select_string"},"import_data_source_id":{"column":"import_data_source_id"},"import_device_type_id":{"column":"import_device_type_id"},"import_device_status_id":{"column":"import_device_status_id"},"inventory_number":{"column":"inventory_number"},"cpu_core_count":{"column":"cpu_core_count"},"ip_address":{"column":"ip_address"},"import_org_level_2_id":{"column":"import_org_level_2_id"},"device_key":{"sql":"CASE WHEN aws_ec2_tags_value IS NOT NULL THEN aws_ec2_tags_value ELSE instanceId END"},"device_name":{"sql":"CASE WHEN aws_ec2_tags_value IS NOT NULL THEN !getNameFromFQDN(aws_ec2_tags_value) ELSE instanceId END"},"fqdn":{"sql":"CASE WHEN aws_ec2_tags_value IS NOT NULL THEN !checkFQDN(aws_ec2_tags_value) ELSE null END"}},"id":407,"type":"map","name":"Transform AWS EC2 device data considering tag information","source":[{"step":406}]},{"mapRemaining":true,"columns":{},"id":497,"type":"map","name":"Combine IAAS and PAAS device data","source":[{"step":401},{"step":407}]},{"conditions":[{"$sql":"[device_key] IS NOT NULL AND [import_device_status_id] = 'active'"}],"top":0,"id":498,"type":"filter","name":"Filter out inactive IAAS and PAAS devices and IAAS and PAAS devices with missing information","source":{"step":497}},{"by":["device_key"],"strategy":{"type":"any"},"id":499,"type":"deduplicate","name":"Deduplicate IAAS and PAAS device data on device key","source":{"step":498}},{"mapRemaining":true,"columns":{"import_id":{"sql":"Lower(!checkFQDN(fqdn))"}},"id":1000,"type":"map","name":"Calculate import id for directory service device data (FQDN)","source":[{"step":199}]},{"column":"import_id","lookup":[{"in":{"step":1000},"key":{"primary":"fqdn","foreign":"fqdn"},"take":{"column":"import_id"}},{"take":{"sql":"Lower(!checkFQDN(fqdn))"}}],"id":2000,"type":"enrich","name":"Enrich import id for software deployment and discovery device data (FQDN)","source":{"step":299}},{"column":"import_id","lookup":[{"in":{"step":1000},"key":{"primary":"fqdn","foreign":"fqdn"},"take":{"column":"import_id"}},{"in":{"step":2000},"key":{"primary":"fqdn","foreign":"fqdn"},"take":{"column":"import_id"}},{"take":{"sql":"Lower(!checkFQDN(fqdn))"}}],"id":3000,"type":"enrich","name":"Enrich import id for hypervisor device data (FQDN)","source":{"step":399}},{"column":"import_id","lookup":[{"in":{"step":1000},"key":{"primary":"fqdn","foreign":"fqdn"},"take":{"column":"import_id"}},{"in":{"step":2000},"key":{"primary":"fqdn","foreign":"fqdn"},"take":{"column":"import_id"}},{"in":{"step":3000},"key":{"primary":"fqdn","foreign":"fqdn"},"take":{"column":"import_id"}},{"take":{"sql":"Lower(!checkFQDN(fqdn))"}}],"id":4000,"type":"enrich","name":"Enrich import id for IAAS and PAAS device data (FQDN)","source":{"step":499}},{"column":"import_id","lookup":[{"in":{"step":2000},"key":{"primary":"device_name","foreign":"device_name"},"take":{"column":"import_id"}},{"in":{"step":3000},"key":{"primary":"device_name","foreign":"device_name"},"take":{"column":"import_id"}},{"in":{"step":4000},"key":{"primary":"device_name","foreign":"device_name"},"take":{"column":"import_id"}},{"take":{"column":"device_name","transform":"Lowercase"}}],"id":1002,"type":"enrich","name":"Enrich import id for directory service device data (DEVICE NAME)","source":{"step":1000}},{"column":"import_id","lookup":[{"in":{"step":1002},"key":{"primary":"device_name","foreign":"device_name"},"take":{"column":"import_id"}},{"in":{"step":3000},"key":{"primary":"device_name","foreign":"device_name"},"take":{"column":"import_id"}},{"in":{"step":4000},"key":{"primary":"device_name","foreign":"device_name"},"take":{"column":"import_id"}},{"take":{"column":"device_name","transform":"Lowercase"}}],"id":2002,"type":"enrich","name":"Enrich import id for software deployment and discovery device data (DEVICE NAME)","source":{"step":2000}},{"column":"import_id","lookup":[{"in":{"step":1002},"key":{"primary":"device_name","foreign":"device_name"},"take":{"column":"import_id"}},{"in":{"step":2002},"key":{"primary":"device_name","foreign":"device_name"},"take":{"column":"import_id"}},{"in":{"step":4000},"key":{"primary":"device_name","foreign":"device_name"},"take":{"column":"import_id"}},{"take":{"column":"device_name","transform":"Lowercase"}}],"id":3002,"type":"enrich","name":"Enrich import id for hypervisor device data (DEVICE NAME)","source":{"step":3000}},{"column":"import_id","lookup":[{"in":{"step":1002},"key":{"primary":"device_name","foreign":"device_name"},"take":{"column":"import_id"}},{"in":{"step":2002},"key":{"primary":"device_name","foreign":"device_name"},"take":{"column":"import_id"}},{"in":{"step":3002},"key":{"primary":"device_name","foreign":"device_name"},"take":{"column":"import_id"}},{"take":{"column":"device_name","transform":"Lowercase"}}],"id":4002,"type":"enrich","name":"Enrich import id for IAAS and PAAS device data (DEVICE NAME)","source":{"step":4000}},{"column":"import_id","lookup":[{"in":{"step":2002},"key":{"primary":"ip_address","foreign":"ip_address"},"take":{"column":"import_id"}},{"in":{"step":3002},"key":{"primary":"ip_address","foreign":"ip_address"},"take":{"column":"import_id"}},{"in":{"step":4002},"key":{"primary":"ip_address","foreign":"ip_address"},"take":{"column":"import_id"}},{"take":{"column":"ip_address","transform":"Lowercase"}}],"id":1998,"type":"enrich","name":"Enrich import id for directory service device data (IP)","source":{"step":1002}},{"column":"import_id","lookup":[{"in":{"step":1998},"key":{"primary":"ip_address","foreign":"ip_address"},"take":{"column":"import_id"}},{"in":{"step":3002},"key":{"primary":"ip_address","foreign":"ip_address"},"take":{"column":"import_id"}},{"in":{"step":4002},"key":{"primary":"ip_address","foreign":"ip_address"},"take":{"column":"import_id"}},{"take":{"column":"ip_address","transform":"Lowercase"}}],"id":2998,"type":"enrich","name":"Enrich import id for software deployment and discovery device data (IP)","source":{"step":2002}},{"column":"import_id","lookup":[{"in":{"step":1998},"key":{"primary":"ip_address","foreign":"ip_address"},"take":{"column":"import_id"}},{"in":{"step":2998},"key":{"primary":"ip_address","foreign":"ip_address"},"take":{"column":"import_id"}},{"in":{"step":4002},"key":{"primary":"ip_address","foreign":"ip_address"},"take":{"column":"import_id"}},{"take":{"column":"ip_address","transform":"Lowercase"}}],"id":3998,"type":"enrich","name":"Enrich import id for hypervisor device data (IP)","source":{"step":3002}},{"column":"import_id","lookup":[{"in":{"step":1998},"key":{"primary":"ip_address","foreign":"ip_address"},"take":{"column":"import_id"}},{"in":{"step":2998},"key":{"primary":"ip_address","foreign":"ip_address"},"take":{"column":"import_id"}},{"in":{"step":3998},"key":{"primary":"ip_address","foreign":"ip_address"},"take":{"column":"import_id"}},{"take":{"column":"ip_address","transform":"Lowercase"}}],"id":4998,"type":"enrich","name":"Enrich import id for IAAS and PAAS device data (IP)","source":{"step":4002}},{"by":["import_id"],"strategy":{"type":"any"},"id":1999,"type":"deduplicate","name":"Deduplicate directory service device data on import id","source":{"step":1998}},{"by":["import_id"],"strategy":{"type":"any"},"id":2999,"type":"deduplicate","name":"Deduplicate software deployment and discovery device data on import id","source":{"step":2998}},{"by":["import_id"],"strategy":{"type":"any"},"id":3999,"type":"deduplicate","name":"Deduplicate hypervisor device data on import id","source":{"step":3998}},{"by":["import_id"],"strategy":{"type":"any"},"id":4999,"type":"deduplicate","name":"Deduplicate IAAS and PAAS device data on import id","source":{"step":4998}},{"column":"import_id","lookup":[{"in":{"step":1999},"key":{"primary":"fqdn","foreign":"fqdn"},"take":{"column":"import_id"}},{"in":{"step":2999},"key":{"primary":"fqdn","foreign":"fqdn"},"take":{"column":"import_id"}},{"in":{"step":3999},"key":{"primary":"fqdn","foreign":"fqdn"},"take":{"column":"import_id"}},{"in":{"step":4999},"key":{"primary":"fqdn","foreign":"fqdn"},"take":{"column":"import_id"}},{"in":{"step":1999},"key":{"primary":"device_name","foreign":"device_name"},"take":{"column":"import_id"}},{"in":{"step":2999},"key":{"primary":"device_name","foreign":"device_name"},"take":{"column":"import_id"}},{"in":{"step":3999},"key":{"primary":"device_name","foreign":"device_name"},"take":{"column":"import_id"}},{"in":{"step":4999},"key":{"primary":"device_name","foreign":"device_name"},"take":{"column":"import_id"}}],"id":9997,"type":"enrich","name":"Enrich import id for custom fields device data (FQDN, DEVICE NAME)","source":{"table":"[DataTransformation-custom_fields_devices]"}},{"conditions":[{"$sql":"[import_id] IS NOT NULL"}],"top":0,"id":9998,"type":"filter","name":"Filter out custom device data with missing information","source":{"step":9997}},{"strategy":"outer","sources":[{"step":1999},{"step":2999},{"step":3999},{"step":4999},{"step":9998}],"on":["import_id"],"conflicts":{"source":{"aggregation":{"type":"concat","separator":" | "}},"__source_id":{"aggregation":{"type":"concat","separator":" | "}},"source_table":{"aggregation":{"type":"concat","separator":" | "}},"__source_select_string":{"aggregation":{"type":"concat","separator":" | "}},"import_data_source_id":{"aggregation":{"type":"coalesce"}},"device_key":{"aggregation":{"type":"coalesce"}},"fqdn":{"aggregation":{"type":"coalesce"}},"device_name":{"aggregation":{"type":"coalesce"}},"hostname":{"aggregation":{"type":"coalesce"}},"department":{"aggregation":{"type":"coalesce"}},"import_org_level_2_id":{"aggregation":{"type":"coalesce"},"priority":[{"step":9998},{"step":2999}]},"import_device_type_id":{"aggregation":{"type":"coalesce"},"priority":[{"step":3999},{"step":4999},{"step":2999},{"step":1999}]},"import_operating_system_id":{"aggregation":{"type":"coalesce"},"priority":[{"step":2999},{"step":4999},{"step":1999},{"step":3999}]},"import_device_status_id":{"aggregation":{"type":"coalesce"},"priority":[{"step":1999},{"step":4999},{"step":3999},{"step":2999}]},"import_domain_id":{"aggregation":{"type":"coalesce"},"priority":[{"step":2999},{"step":1999},{"step":3999},{"step":9998}]},"import_location_id":{"aggregation":{"type":"coalesce"}},"import_user_id":{"aggregation":{"type":"coalesce"},"priority":[{"step":9998},{"step":2999}]},"inventory_date":{"aggregation":{"type":"coalesce"},"priority":[{"step":2999},{"step":9998}]},"installation_date":{"aggregation":{"type":"coalesce"},"priority":[{"step":2999},{"step":9998}]},"last_logon_timestamp":{"aggregation":{"type":"coalesce"},"priority":[{"step":1999},{"step":9998}]},"device_manufacturer":{"aggregation":{"type":"coalesce"},"priority":[{"step":2999},{"step":3999},{"step":9998}]},"device_model":{"aggregation":{"type":"coalesce"},"priority":[{"step":2999},{"step":3999},{"step":9998}]},"serial_number":{"aggregation":{"type":"coalesce"},"priority":[{"step":2999},{"step":3999},{"step":9998}]},"ip_address":{"aggregation":{"type":"coalesce"},"priority":[{"step":2999},{"step":3999},{"step":4999},{"step":9998}]},"mac_address":{"aggregation":{"type":"coalesce"},"priority":[{"step":2999},{"step":3999},{"step":4999},{"step":9998}]},"cpu_chip_count":{"aggregation":{"type":"coalesce"},"priority":[{"step":3999},{"step":2999},{"step":9998}]},"cpu_core_count":{"aggregation":{"type":"coalesce"},"priority":[{"step":3999},{"step":2999},{"step":4999},{"step":9998}]},"cpu_speed":{"aggregation":{"type":"coalesce"},"priority":[{"step":3999},{"step":2999},{"step":9998}]},"import_cpu_type_id":{"aggregation":{"type":"coalesce"},"priority":[{"step":3999},{"step":2999},{"step":4999},{"step":9998}]},"hyper_threading_act":{"aggregation":{"type":"coalesce"},"priority":[{"step":2999},{"step":9998}]},"cpu_thread_count":{"aggregation":{"type":"coalesce"},"priority":[{"step":2999},{"step":3999},{"step":4999},{"step":9998}]},"limit_cpu_core_count":{"aggregation":{"type":"coalesce"},"priority":[{"step":2999},{"step":9998}]},"cpu_core_count_limited":{"aggregation":{"type":"coalesce"},"priority":[{"step":2999},{"step":9998}]},"ram":{"aggregation":{"type":"coalesce"},"priority":[{"step":2999},{"step":3999},{"step":9998}]},"storage":{"aggregation":{"type":"coalesce"},"priority":[{"step":2999},{"step":3999},{"step":9998}]},"graphics":{"aggregation":{"type":"coalesce"},"priority":[{"step":2999},{"step":9998}]},"network":{"aggregation":{"type":"coalesce"},"priority":[{"step":2999},{"step":9998}]},"bios":{"aggregation":{"type":"coalesce"},"priority":[{"step":2999},{"step":3999},{"step":9998}]},"uuid":{"aggregation":{"type":"coalesce"},"priority":[{"step":2999},{"step":3999},{"step":9998}]},"inventory_number":{"aggregation":{"type":"coalesce"},"priority":[{"step":4999},{"step":2999},{"step":9998}]}},"id":10000,"type":"join","name":"Calculate result device data"},{"conditions":[{"$sql":"[import_id] IS NOT NULL"}],"top":0,"id":10002,"type":"filter","name":"Filter result device data","source":{"step":10000}},{"mapRemaining":false,"columns":{"import_id":{"type":"nvarchar(255)","column":"import_id"},"import_data_source_id":{"type":"nvarchar(255)","value":"DataHub"},"source":{"type":"nvarchar(255)","column":"source"},"__source_id":{"type":"nvarchar(255)","column":"__source_id"},"source_table":{"type":"nvarchar(255)","column":"source_table"},"__source_select_string":{"type":"nvarchar(max)","column":"__source_select_string"},"device_key":{"type":"nvarchar(255)","column":"device_key"},"fqdn":{"type":"nvarchar(255)","column":"fqdn"},"device_name":{"type":"nvarchar(255)","column":"device_name"},"hostname":{"type":"nvarchar(255)","fallback":{"value":null},"column":"hostname"},"department":{"type":"nvarchar(255)","fallback":{"value":null},"column":"department"},"import_org_level_2_id":{"type":"nvarchar(255)","fallback":{"value":"unknown"},"column":"import_org_level_2_id"},"import_device_type_id":{"type":"nvarchar(255)","column":"import_device_type_id"},"import_operating_system_id":{"type":"nvarchar(255)","column":"import_operating_system_id"},"import_device_status_id":{"type":"nvarchar(255)","fallback":{"value":null},"column":"import_device_status_id"},"import_domain_id":{"type":"nvarchar(255)","column":"import_domain_id"},"import_location_id":{"type":"nvarchar(255)","fallback":{"value":null},"column":"import_location_id"},"import_user_id":{"type":"nvarchar(255)","fallback":{"value":null},"column":"import_user_id"},"inventory_date":{"type":"nvarchar(255)","column":"inventory_date"},"installation_date":{"type":"nvarchar(255)","column":"installation_date"},"last_logon_timestamp":{"type":"nvarchar(255)","fallback":{"value":null},"column":"last_logon_timestamp"},"device_manufacturer":{"type":"nvarchar(255)","column":"device_manufacturer"},"device_model":{"type":"nvarchar(255)","column":"device_model"},"serial_number":{"type":"nvarchar(255)","column":"serial_number"},"ip_address":{"type":"nvarchar(510)","column":"ip_address"},"mac_address":{"type":"nvarchar(255)","column":"mac_address"},"cpu_chip_count":{"type":"NUM(13,2)","column":"cpu_chip_count","compact":true},"cpu_speed":{"type":"INT","column":"cpu_speed","compact":true},"import_cpu_type_id":{"type":"nvarchar(255)","column":"import_cpu_type_id","compact":true},"cpu_core_count":{"type":"NUM(13,2)","column":"cpu_core_count"},"hyper_threading_act":{"type":"nvarchar(255)","fallback":{"value":null},"column":"hyper_threading_act"},"cpu_thread_count":{"type":"NUM(13,2)","fallback":{"value":null},"column":"cpu_thread_count"},"limit_cpu_core_count":{"type":"Bit","fallback":{"value":null},"column":"limit_cpu_core_count"},"cpu_core_count_limited":{"type":"NUM(13,2)","fallback":{"value":null},"column":"cpu_core_count_limited"},"ram":{"type":"INT","fallback":{"value":null},"column":"ram"},"storage":{"type":"INT","fallback":{"value":null},"column":"storage"},"graphics":{"type":"nvarchar(255)","fallback":{"value":null},"column":"graphics"},"network":{"type":"nvarchar(510)","fallback":{"value":null},"column":"network"},"bios":{"type":"nvarchar(255)","fallback":{"value":null},"column":"bios"},"uuid":{"type":"nvarchar(255)","fallback":{"value":null},"column":"uuid"},"inventory_number":{"type":"nvarchar(255)","fallback":{"value":null},"column":"inventory_number"}},"id":10001,"type":"map","name":"Format result device data","source":[{"step":10002}],"target":{"name":"result_devices"}},{"mapRemaining":false,"columns":{"import_data_source_id":{"value":"vCenter"},"source":{"value":"vCenter"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_vcenter_vms] WHERE __source_id =' || __source_id"},"import_device_rel_type_id":{"value":"is_running_on_esx"},"from_device_id":{"sql":"!GetKeyFromFqdnAndName(fqdn, name)"},"to_device_host_id":{"column":"vmHost"},"to_device_cluster_id":{"column":"cluster"},"powerState":{"column":"powerState"}},"id":20000,"type":"map","name":"Add relation information for vCenter VM data","source":[{"step":302}]},{"mapRemaining":false,"columns":{"to_device_host_id":{"column":"name"},"to_device_cluster_id":{"column":"cluster"}},"id":20001,"type":"map","name":"Add relation information for vCenter host data","source":[{"step":300}]},{"strategy":"left","sources":[{"step":20000},{"step":20001}],"on":["to_device_host_id"],"conflicts":{"to_device_cluster_id":{"aggregation":{"type":"coalesce"}}},"id":20002,"type":"join","name":"Calculate relation for vCenter device data"},{"mapRemaining":false,"columns":{"import_data_source_id":{"column":"import_data_source_id"},"source":{"column":"source"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"column":"__source_select_string"},"import_device_rel_type_id":{"column":"import_device_rel_type_id"},"from_device_id":{"column":"from_device_id"},"to_device_id":{"sql":"CASE WHEN to_device_cluster_id IS NOT NULL AND to_device_cluster_id NOT LIKE 'NULL' THEN to_device_cluster_id ELSE to_device_host_id END"},"to_device_host_id":{"column":"to_device_host_id"},"powerState":{"column":"powerState"}},"id":20003,"type":"map","name":"Add metadata and calculate relation for vCenter device data","source":[{"step":20002}]},{"mapRemaining":false,"columns":{"import_data_source_id":{"value":"vCenter"},"source":{"value":"vCenter"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_vcenter_vmhosts] WHERE __source_id =' || __source_id"},"import_device_rel_type_id":{"value":"is_part_of_esx_cluster"},"from_device_id":{"column":"name"},"to_device_id":{"column":"cluster","compact":true},"powerState":{"column":"powerState"}},"id":20004,"type":"map","name":"Add metadata and relation information for vCenter host device data","source":[{"step":300}]},{"conditions":[{"$sql":"[to_device_id] IS NOT NULL"}],"top":0,"id":20005,"type":"filter","name":"Filter out vCenter host devices with missing information","source":{"step":20004}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":20011,"type":"map","name":"Add metadata for RV device relation data","source":[{"table":"InventoryDeviceRelations*"}],"target":{"name":"__snapshot_InventoryDeviceRelations"}},{"mapRemaining":false,"columns":{"import_data_source_id":{"value":"RV"},"source":{"value":"RV"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_InventoryDeviceRelations] WHERE __source_id =' || __source_id"},"import_device_rel_type_id":{"column":"import_device_rel_type_id"},"from_device_id":{"column":"import_from_device_id"},"to_device_id":{"column":"import_to_device_id"},"to_device_host_id":{"column":"import_to_device_host_id"},"powerState":{"column":"tmp_powerstate"}},"id":20012,"type":"map","name":"Transform RV device relation data","source":[{"step":20011}]},{"mapRemaining":true,"columns":{},"id":210102,"type":"map","name":"Combine relation data","source":[{"step":20003},{"step":20005},{"step":20012}]},{"column":"import_from_device_id","lookup":[{"in":{"step":10001},"key":{"primary":"from_device_id","foreign":"device_name"},"take":{"column":"import_id"}},{"take":{"column":"from_device_id"}}],"id":210103,"type":"enrich","name":"Enrich import from device id for relation data","source":{"step":210102}},{"column":"import_to_device_id","lookup":[{"in":{"step":10001},"key":{"primary":"to_device_id","foreign":"device_name"},"take":{"column":"import_id"}},{"take":{"column":"to_device_id"}}],"id":210104,"type":"enrich","name":"Enrich import to device id for relation data","source":{"step":210103}},{"column":"import_to_device_host_id","lookup":[{"in":{"step":10001},"key":{"primary":"to_device_host_id","foreign":"device_name"},"take":{"column":"import_id"}},{"take":{"column":"to_device_host_id"}}],"id":210105,"type":"enrich","name":"Enrich import to device host id for relation data","source":{"step":210104}},{"column":"import_from_device_id_missing","lookup":[{"in":{"step":10001},"key":{"primary":"import_from_device_id","foreign":"import_id"},"take":{"column":"import_id"}},{"take":{"sql":"'true'"}}],"id":210106,"type":"enrich","name":"Identify not needed device relation data","source":{"step":210105}},{"column":"import_to_device_id_missing","lookup":[{"in":{"step":10001},"key":{"primary":"import_to_device_id","foreign":"import_id"},"take":{"column":"import_id"}},{"take":{"sql":"'true'"}}],"id":210107,"type":"enrich","name":"Identify not needed device relation data","source":{"step":210106}},{"conditions":[{"$sql":"[import_from_device_id_missing] <> 'true' AND [import_to_device_id_missing] <> 'true'"}],"top":0,"id":210108,"type":"filter","name":"Filter out not needed device relation data","source":{"step":210107}},{"mapRemaining":false,"columns":{"source":{"type":"nvarchar(255)","column":"source"},"__source_id":{"type":"nvarchar(255)","column":"__source_id"},"source_table":{"type":"nvarchar(255)","column":"source_table"},"__source_select_string":{"type":"nvarchar(max)","column":"__source_select_string"},"import_id":{"type":"nvarchar(510)","sql":"import_from_device_id || '_' || import_to_device_id"},"import_data_source_id":{"type":"nvarchar(255)","column":"import_data_source_id"},"import_device_rel_type_id":{"type":"nvarchar(255)","column":"import_device_rel_type_id"},"import_from_device_id":{"type":"nvarchar(255)","column":"import_from_device_id"},"import_to_device_id":{"type":"nvarchar(255)","column":"import_to_device_id"},"import_to_device_host_id":{"type":"nvarchar(255)","fallback":{"value":null},"column":"import_to_device_host_id"},"powerState":{"type":"nvarchar(255)","fallback":{"value":null},"column":"powerState"}},"id":20007,"type":"map","name":"Format result relation data","source":[{"step":210108}],"target":{"name":"result_device_relations"}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":100100,"type":"map","name":"Add metadata for active directory user data","source":[{"table":"ActiveDirectoryUsers*"}],"target":{"name":"__snapshot_ActiveDirectoryUsers"}},{"mapRemaining":false,"columns":{"import_data_source_id":{"value":"AD"},"source":{"value":"AD"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_ActiveDirectoryUsers] WHERE __source_id =' || __source_id"},"id":{"value":null},"user_key":{"sql":"COALESCE([msDS-PrincipalName], userPrincipalName)"},"login":{"sql":"COALESCE([msDS-PrincipalName], userPrincipalName)"},"last_name":{"sql":"!getLastName(displayName, givenName)"},"first_name":{"sql":"!getName(givenName)"},"email":{"sql":"!checkUnknownValue(userPrincipalName)"},"is_active":{"sql":"!isActive(userAccountControl)"},"userPrincipalName":{"sql":"COALESCE(userPrincipalName, [msDS-PrincipalName])"}},"id":100101,"type":"map","name":"Transform active directory user data","source":[{"step":100100}]},{"mapRemaining":true,"columns":{},"id":100197,"type":"map","name":"Combine directory service user data","source":[{"step":100101}]},{"conditions":[{"$sql":"[user_key] IS NOT NULL"}],"top":0,"id":100198,"type":"filter","name":"Filter out directory service users with missing information","source":{"step":100197}},{"by":["user_key"],"strategy":{"type":"any"},"id":100199,"type":"deduplicate","name":"Deduplicate directory service user data on email","source":{"step":100198}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":200100,"type":"map","name":"Add metadata for SCCM WMI user data","source":[{"table":"SCCM_WMI*-UserDeviceRelations"}],"target":{"name":"__snapshot_SCCM_WMI_UserDeviceRelations"}},{"mapRemaining":false,"columns":{"import_data_source_id":{"value":"SCCM_WMI"},"source":{"value":"SCCM_WMI"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_SCCM_WMI_UserDeviceRelations] WHERE __source_id =' || __source_id"},"user_key":{"column":"import_user_id"},"login":{"column":"import_user_id"},"last_name":{"value":"unknown"},"first_name":{"value":"unknown"},"email":{"sql":"!checkUnknownValue(user_principal_name)"},"is_active":{"value":1},"userPrincipalName":{"column":"user_principal_name"}},"id":200101,"type":"map","name":"Transform SCCM WMI user data","source":[{"step":200100}]},{"conditions":[{"$sql":"[user_key] IS NOT NULL"}],"top":0,"id":200198,"type":"filter","name":"Filter SCCM WMI out users with missing information","source":{"step":200101}},{"by":["user_key"],"strategy":{"type":"any"},"id":200199,"type":"deduplicate","name":"Deduplicate SCCM WMI user data on email","source":{"step":200198}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":100200,"type":"map","name":"Add metadata for Office 365 user data","source":[{"table":"Office365Data*-users"}],"target":{"name":"__snapshot_Office365Data_users"}},{"mapRemaining":false,"columns":{"import_data_source_id":{"value":"m365"},"source":{"value":"m365"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_Office365Data_users] WHERE __source_id =' || __source_id"},"id":{"column":"id"},"user_key":{"column":"userPrincipalName"},"login":{"column":"userPrincipalName"},"userPrincipalName":{"column":"userPrincipalName"},"userPrincipalName_MD5":{"column":"userPrincipalName","transform":"Md5"},"last_name":{"sql":"!getName(surname)"},"first_name":{"sql":"!getName(givenname)"},"email":{"sql":"COALESCE(mail, userPrincipalName)"},"is_active":{"value":1}},"id":100201,"type":"map","name":"Transform Office 365 user data","source":[{"step":100200}]},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":100202,"type":"map","name":"Add metadata for Adobe AC user data","source":[{"table":"adobe_ac*-users"}],"target":{"name":"__snapshot_adobe_ac_users"}},{"mapRemaining":false,"columns":{"import_data_source_id":{"value":"adobe_admin_console"},"source":{"value":"adobe_admin_console"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_adobe_ac_users] WHERE __source_id =' || __source_id"},"id":{"value":null},"user_key":{"column":"username"},"login":{"column":"username"},"userPrincipalName":{"column":"username"},"last_name":{"sql":"!getName(lastname)"},"first_name":{"sql":"!getName(firstname)"},"email":{"column":"email"},"is_active":{"sql":"CASE WHEN status = 'active' THEN 1 ELSE 0 END"}},"id":100203,"type":"map","name":"Transform Adobe AC user data","source":[{"step":100202}]},{"mapRemaining":true,"columns":{},"id":100297,"type":"map","name":"Combine SAAS user data","source":[{"step":100201},{"step":100203}]},{"conditions":[{"$sql":"[user_key] IS NOT NULL"}],"top":0,"id":100298,"type":"filter","name":"Filter out SAAS users with missing information","source":{"step":100297}},{"by":["user_key"],"strategy":{"type":"any"},"id":100299,"type":"deduplicate","name":"Deduplicate SAAS user data on user key","source":{"step":100298}},{"mapRemaining":true,"columns":{"import_id":{"sql":"Lower(login)"}},"id":1001998,"type":"map","name":"Calculate import id for directory service user data (login)","source":[{"step":100199}]},{"column":"import_id","lookup":[{"in":{"step":1001998},"key":{"primary":"userPrincipalName","foreign":"userPrincipalName"},"take":{"column":"import_id"}},{"take":{"sql":"Lower(login)"}}],"id":1002998,"type":"enrich","name":"Enrich import id for SAAS user data (login)","source":{"step":100299}},{"column":"import_id","lookup":[{"in":{"step":1001998},"key":{"primary":"userPrincipalName","foreign":"userPrincipalName"},"take":{"column":"import_id"}},{"in":{"step":1002998},"key":{"primary":"userPrincipalName","foreign":"userPrincipalName"},"take":{"column":"import_id"}},{"take":{"sql":"Lower(login)"}}],"id":2002998,"type":"enrich","name":"Enrich import id for SCCM WMI user (login)","source":{"step":200199}},{"column":"import_id","lookup":[{"in":{"step":1001998},"key":{"primary":"userPrincipalName","foreign":"userPrincipalName"},"take":{"column":"import_id"}},{"in":{"step":1002998},"key":{"primary":"userPrincipalName","foreign":"userPrincipalName"},"take":{"column":"import_id"}},{"in":{"step":2002998},"key":{"primary":"userPrincipalName","foreign":"userPrincipalName"},"take":{"column":"import_id"}}],"id":1009997,"type":"enrich","name":"Enrich import id for custom fields user data (login)","source":{"table":"[DataTransformation-custom_fields_users]"}},{"conditions":[{"$sql":"[import_id] IS NOT NULL"}],"top":0,"id":1009998,"type":"filter","name":"Filter out custom user data with missing information","source":{"step":1009997}},{"strategy":"outer","sources":[{"step":1001998},{"step":1002998},{"step":1009998},{"step":2002998}],"on":["import_id"],"conflicts":{"source":{"aggregation":{"type":"concat","separator":" | "}},"__source_id":{"aggregation":{"type":"concat","separator":" | "}},"source_table":{"aggregation":{"type":"concat","separator":" | "}},"__source_select_string":{"aggregation":{"type":"concat","separator":" | "}},"import_data_source_id":{"aggregation":{"type":"coalesce"}},"user_key":{"aggregation":{"type":"coalesce"}},"login":{"aggregation":{"type":"coalesce"}},"userPrincipalName":{"aggregation":{"type":"coalesce"}},"import_location_id":{"aggregation":{"type":"coalesce"},"priority":[{"step":1009998}]},"import_level_2_id":{"aggregation":{"type":"coalesce"},"priority":[{"step":1009998}]},"last_name":{"aggregation":{"type":"coalesce"}},"first_name":{"aggregation":{"type":"coalesce"}},"email":{"aggregation":{"type":"coalesce"}},"is_active":{"aggregation":{"type":"coalesce"}}},"id":10010000,"type":"join","name":"Calculate result user data"},{"conditions":[{"$sql":"[import_id] IS NOT NULL"}],"top":0,"id":10010002,"type":"filter","name":"Filter out users with missing information","source":{"step":10010000}},{"mapRemaining":false,"columns":{"source":{"type":"nvarchar(255)","column":"source"},"__source_id":{"type":"nvarchar(255)","column":"__source_id"},"source_table":{"type":"nvarchar(255)","column":"source_table"},"__source_select_string":{"type":"nvarchar(max)","column":"__source_select_string"},"import_id":{"type":"nvarchar(255)","column":"import_id"},"import_data_source_id":{"type":"nvarchar(255)","value":"DataHub"},"user_key":{"type":"nvarchar(255)","column":"user_key"},"login":{"type":"nvarchar(255)","column":"login"},"userPrincipalName":{"type":"nvarchar(255)","column":"userPrincipalName"},"import_location_id":{"type":"nvarchar(255)","fallback":{"value":null},"column":"import_location_id"},"import_level_2_id":{"type":"nvarchar(255)","fallback":{"value":"unknown"},"column":"import_level_2_id"},"last_name":{"type":"nvarchar(255)","fallback":{"value":"unknown"},"column":"last_name"},"first_name":{"type":"nvarchar(255)","fallback":{"value":"unknown"},"column":"first_name"},"email":{"type":"nvarchar(255)","column":"email"},"is_active":{"type":"bit","column":"is_active"}},"id":10010001,"type":"map","name":"Format result user data","source":[{"step":10010002}],"target":{"name":"result_users"}},{"mapRemaining":false,"columns":{"import_data_source_id":{"column":"import_data_source_id"},"source":{"column":"source"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"column":"__source_select_string"},"userPrincipalName":{"column":"userPrincipalName"},"import_id":{"sql":"COALESCE(id, source || '_' || userPrincipalName)"},"account":{"column":"userPrincipalName"},"import_provider_instance_id":{"column":"source"}},"id":10020001,"type":"map","name":"Calculate user account data","source":[{"step":100199},{"step":100299}]},{"column":"import_user_id","lookup":[{"in":{"step":10010002},"key":{"primary":"userPrincipalName","foreign":"userPrincipalName"},"take":{"column":"import_id"}},{"take":{"sql":"Lower(userPrincipalName)"}}],"id":10029998,"type":"enrich","name":"Enrich import user id for user account data (userPrincipalName)","source":{"step":10020001}},{"conditions":[{"$sql":"[import_user_id] IS NOT NULL"}],"top":0,"id":10029997,"type":"filter","name":"Filter out user accounts with missing information","source":{"step":10029998}},{"by":["import_id"],"strategy":{"type":"any"},"id":10029999,"type":"deduplicate","name":"Deduplicate user account data on import id","source":{"step":10029997}},{"mapRemaining":false,"columns":{"source":{"type":"nvarchar(255)","column":"source"},"__source_id":{"type":"nvarchar(255)","column":"__source_id"},"source_table":{"type":"nvarchar(255)","column":"source_table"},"__source_select_string":{"type":"nvarchar(max)","column":"__source_select_string"},"import_id":{"type":"nvarchar(255)","column":"import_id"},"import_data_source_id":{"type":"nvarchar(255)","column":"import_data_source_id"},"import_provider_instance_id":{"type":"nvarchar(255)","column":"import_provider_instance_id"},"import_user_id":{"type":"nvarchar(255)","column":"import_user_id"},"userPrincipalName":{"type":"nvarchar(255)","column":"userPrincipalName"},"account":{"type":"nvarchar(255)","column":"account"}},"id":10030000,"type":"map","name":"Format result user account data","source":[{"step":10029999}],"target":{"name":"result_accounts"}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":10030011,"type":"map","name":"Add metadata for ADOBE AC group data","source":[{"table":"adobe_ac*-groups"}],"target":{"name":"__snapshot_adobe_ac_groups"}},{"by":["groupId"],"strategy":{"type":"any"},"id":10030001,"type":"deduplicate","name":"Deduplicate ADOBE AC group data","source":{"step":10030011}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":10030002,"type":"map","name":"Add metadata for ADOBE AC products data","source":[{"table":"adobe_ac*-products"}],"target":{"name":"__snapshot_adobe_ac_products"}},{"mapRemaining":true,"columns":{"source":{"value":"adobe_ac_users"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_adobe_ac_users] WHERE __source_id =' || __source_id"},"user_type":{"column":"type"}},"id":10030202,"type":"map","name":"Add metadata for ADOBE AC user data and transform data","source":[{"step":100202}]},{"mapRemaining":true,"columns":{"groupname":{"column":"groupname"},"groups":{"column":"groupname"},"shortname":{"sql":"REPLACE([productName],' (ETLA)', '')"},"longname":{"sql":"REPLACE([productName],' (ETLA)', '')"},"group_type":{"column":"type"},"source":{"value":"adobe_ac_groups"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_adobe_ac_groups] WHERE __source_id =' || __source_id"}},"id":10030003,"type":"map","name":"Add metadata for ADOBE AC group data and transform data","source":[{"step":10030001}]},{"mapRemaining":true,"columns":{"shortname":{"column":"shortname"},"source":{"value":"adobe_ac_products"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_adobe_ac_products] WHERE __source_id =' || __source_id"}},"id":10030004,"type":"map","name":"Add metadata for ADOBE AC product data and transform data (shortname)","source":[{"step":10030002}]},{"mapRemaining":true,"columns":{"longname":{"column":"longname"},"source":{"value":"adobe_ac_products"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_adobe_ac_products] WHERE __source_id =' || __source_id"}},"id":10030005,"type":"map","name":"Add metadata for ADOBE AC product data and transform data (longname)","source":[{"step":10030002}]},{"strategy":"inner","sources":[{"step":10030202},{"step":10030003}],"on":["groups"],"conflicts":{"source":{"aggregation":{"type":"concat","separator":" | "}},"__source_id":{"aggregation":{"type":"concat","separator":" | "}},"source_table":{"aggregation":{"type":"concat","separator":" | "}},"__source_select_string":{"aggregation":{"type":"concat","separator":" | "}}},"id":10030006,"type":"join","name":"Combine ADOBE AC user and group data"},{"strategy":"left","sources":[{"step":10030006},{"step":10030004}],"on":["shortname"],"conflicts":{"source":{"aggregation":{"type":"concat","separator":" | "}},"__source_id":{"aggregation":{"type":"concat","separator":" | "}},"source_table":{"aggregation":{"type":"concat","separator":" | "}},"__source_select_string":{"aggregation":{"type":"concat","separator":" | "}}},"id":10030007,"type":"join","name":"Combine ADOBE AC user, group and products data (shortname)"},{"strategy":"left","sources":[{"step":10030007},{"step":10030005}],"on":["longname"],"conflicts":{"source":{"aggregation":{"type":"concat","separator":" | "}},"__source_id":{"aggregation":{"type":"concat","separator":" | "}},"source_table":{"aggregation":{"type":"concat","separator":" | "}},"__source_select_string":{"aggregation":{"type":"concat","separator":" | "}}},"id":10030008,"type":"join","name":"Combine ADOBE AC user, group and products data (longname)"},{"conditions":[{"$sql":"[productName] IS NOT NULL AND [group_type] IS 'PRODUCT_PROFILE'"}],"top":0,"id":10030019,"type":"filter","name":"Filter Adobe AC data","source":{"step":10030008}},{"by":["username","groupId","id"],"strategy":{"type":"any"},"id":10030009,"type":"deduplicate","name":"Deduplicate Adobe AC data","source":{"step":10030019}},{"mapRemaining":false,"columns":{"source":{"type":"nvarchar(255)","column":"source"},"__source_id":{"type":"nvarchar(255)","column":"__source_id"},"source_table":{"type":"nvarchar(255)","column":"source_table"},"__source_select_string":{"type":"nvarchar(max)","column":"__source_select_string"},"import_id":{"type":"nvarchar(255)","sql":"productname || '_' || user_type || '_' || username || '_' || groupname"},"import_account_id":{"type":"nvarchar(255)","sql":"LOWER ('Adobe Admin Console_' || username)"},"generic_key":{"type":"nvarchar(255)","sql":"COALESCE(code, productname)"},"quantity":{"value":1},"import_data_source_id":{"type":"nvarchar(255)","value":"adobe_admin_console"},"publisher":{"type":"nvarchar(255)","value":"Adobe"},"product":{"type":"nvarchar(255)","column":"productname"},"product_version":{"type":"nvarchar(255)","sql":"'contract_type: ' || user_type"},"description":{"type":"nvarchar(255)","value":"Adobe Admin Console"}},"id":10039998,"type":"map","name":"Transform and format raw generic Adobe AC data","source":[{"step":10030009}]},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":61001,"type":"map","name":"Add metadata for O365 assigned license for user data","source":[{"table":"Office365Data*-assigned_licenses_for_users"}],"target":{"name":"__snapshot_Office365Data_assigned_licenses_for_users"}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":61002,"type":"map","name":"Add metadata for O365 subscribed skus data","source":[{"table":"Office365Data*-subscribed_skus"}],"target":{"name":"__snapshot_Office365Data_subscribed_skus"}},{"mapRemaining":true,"columns":{"userid":{"column":"userid"},"id":{"column":"userid"},"source":{"value":"O365_lic"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_Office365Data_assigned_licenses_for_users] WHERE __source_id =' || __source_id"}},"id":61003,"type":"map","name":"Transform O365 assigned license for user data","source":[{"step":61001}]},{"mapRemaining":true,"columns":{"source":{"value":"O365_sku"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_Office365Data_subscribed_skus] WHERE __source_id =' || __source_id"}},"id":61004,"type":"map","name":"Transform O365 subscribed skus data","source":[{"step":61002}]},{"strategy":"inner","sources":[{"step":100201},{"step":61003}],"on":["id"],"conflicts":{"source":{"aggregation":{"type":"concat","separator":" | "}},"__source_id":{"aggregation":{"type":"concat","separator":" | "}},"source_table":{"aggregation":{"type":"concat","separator":" | "}},"__source_select_string":{"aggregation":{"type":"concat","separator":" | "}}},"id":61005,"type":"join","name":"Combine O365 user and assigned license for user data"},{"strategy":"inner","sources":[{"step":61005},{"step":61004}],"on":["skuid"],"conflicts":{"source":{"aggregation":{"type":"concat","separator":" | "}},"__source_id":{"aggregation":{"type":"concat","separator":" | "}},"source_table":{"aggregation":{"type":"concat","separator":" | "}},"__source_select_string":{"aggregation":{"type":"concat","separator":" | "}}},"id":61006,"type":"join","name":"Add O365 subscribed skus data"},{"by":["skupartnumber","userprincipalname"],"strategy":{"type":"any"},"id":61007,"type":"deduplicate","name":"Deduplicate O365 inv raw generic licenses data","source":{"step":61006}},{"mapRemaining":false,"columns":{"source":{"type":"nvarchar(255)","column":"source"},"__source_id":{"type":"nvarchar(255)","column":"__source_id"},"source_table":{"type":"nvarchar(255)","column":"source_table"},"__source_select_string":{"type":"nvarchar(max)","column":"__source_select_string"},"import_id":{"sql":"MD5( LOWER(skupartnumber || userprincipalname))"},"import_account_id":{"type":"nvarchar(255)","sql":"'O365_' || userPrincipalName "},"generic_key":{"type":"nvarchar(255)","column":"skuid"},"quantity":{"value":1},"import_data_source_id":{"type":"nvarchar(255)","value":"m365"},"publisher":{"type":"nvarchar(255)","value":"Microsoft"},"product":{"type":"nvarchar(255)","column":"skupartnumber"},"description":{"type":"nvarchar(255)","value":"Microsoft 365 Subscription"}},"id":61008,"type":"map","name":"Add metadata, transform and format O365 inv raw generic licenses data","source":[{"step":61007}]},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":61021,"type":"map","name":"Add metadata for O365 service plans for subscribed skus data","source":[{"table":"Office365Data*-service_plans_for_subscribed_skus"}],"target":{"name":"__snapshot_Office365Data_service_plans_for_subscribed_skus"}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":61022,"type":"map","name":"Add metadata for O365 assigned plans for users data","source":[{"table":"Office365Data*-assigned_plans_for_users"}],"target":{"name":"__snapshot_Office365Data_assigned_plans_for_users"}},{"mapRemaining":true,"columns":{"source":{"value":"O365_srv"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_Office365Data_service_plans_for_subscribed_skus] WHERE __source_id =' || __source_id"}},"id":61023,"type":"map","name":"Transform O365 service plans for subscribed skus data","source":[{"step":61021}]},{"mapRemaining":true,"columns":{"userid":{"column":"userid"},"id":{"column":"userid"},"source":{"value":"O365_upln"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_Office365Data_assigned_plans_for_users] WHERE __source_id =' || __source_id"}},"id":61024,"type":"map","name":"Transform O365 assigned plans for users data","source":[{"step":61022}]},{"strategy":"inner","sources":[{"step":100201},{"step":61024}],"on":["id"],"conflicts":{"source":{"aggregation":{"type":"concat","separator":" | "}},"__source_id":{"aggregation":{"type":"concat","separator":" | "}},"source_table":{"aggregation":{"type":"concat","separator":" | "}},"__source_select_string":{"aggregation":{"type":"concat","separator":" | "}}},"id":61025,"type":"join","name":"Combine O365 user and assigned plans for users data"},{"strategy":"inner","sources":[{"step":61025},{"step":61023}],"on":["serviceplanid"],"conflicts":{"source":{"aggregation":{"type":"concat","separator":" | "}},"__source_id":{"aggregation":{"type":"concat","separator":" | "}},"source_table":{"aggregation":{"type":"concat","separator":" | "}},"__source_select_string":{"aggregation":{"type":"concat","separator":" | "}}},"id":61026,"type":"join","name":"Add O365 service plans for subscribed skus data"},{"by":["serviceplanname","userprincipalname","service"],"strategy":{"type":"maxvalue","column":"assigneddatetime"},"id":61027,"type":"deduplicate","name":"Find most recent O365 inv raw generic user activity data","source":{"step":61026}},{"conditions":[{"$sql":"[capabilitystatus] != 'Deleted'"}],"top":0,"id":61028,"type":"filter","name":"Filter out O365 inv raw generic data with capability status having deleted","source":{"step":61027}},{"mapRemaining":false,"columns":{"source":{"type":"nvarchar(255)","column":"source"},"__source_id":{"type":"nvarchar(255)","column":"__source_id"},"source_table":{"type":"nvarchar(255)","column":"source_table"},"__source_select_string":{"type":"nvarchar(max)","column":"__source_select_string"},"import_id":{"sql":"MD5( LOWER(serviceplanname || userprincipalname || service || COALESCE(substr(assigneddatetime, 1, 10), '')))"},"import_account_id":{"type":"nvarchar(255)","sql":"'O365_' || userPrincipalName "},"generic_key":{"type":"nvarchar(255)","column":"serviceplanid"},"quantity":{"value":1},"import_data_source_id":{"type":"nvarchar(255)","value":"m365"},"publisher":{"type":"nvarchar(255)","value":"Microsoft"},"product":{"type":"nvarchar(255)","column":"serviceplanname"},"description":{"type":"nvarchar(255)","value":"Microsoft 365 Application"},"installation_date":{"type":"nvarchar(255)","sql":"substr(assigneddatetime, 1, 10)"}},"id":61029,"type":"map","name":"Add metadata, transform and format O365 inv raw generic user activity data","source":[{"step":61028}]},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":61031,"type":"map","name":"Add metadata for O365 user activity data","source":[{"table":"Office365Data*-user_activity"}],"target":{"name":"__snapshot_Office365Data_user_activity"}},{"mapRemaining":false,"columns":{"source":{"value":"O365_ua"},"source_table":{"column":"source_table"},"__source_id":{"column":"__source_id"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_Office365Data_user_activity] WHERE __source_id =' || __source_id"},"userprincipalname":{"column":"userprincipalname"},"userPrincipalName_MD5":{"column":"userprincipalname"},"product":{"value":"Exchange"},"last_usage":{"column":"exchangeLastActivityDate"},"installation_date":{"column":"exchangeLicenseAssignDate"},"hasexchangelicense":{"column":"hasexchangelicense"}},"id":61032,"type":"map","name":"Transform O365 user activity data (Exchange)","source":[{"step":61031}]},{"conditions":[{"$sql":"[hasexchangelicense] == 'true'"}],"top":0,"id":61033,"type":"filter","name":"Find user activity data with an Exchange license","source":{"step":61032}},{"mapRemaining":false,"columns":{"source":{"value":"O365_ua"},"source_table":{"column":"source_table"},"__source_id":{"column":"__source_id"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_Office365Data_user_activity] WHERE __source_id =' || __source_id"},"userprincipalname":{"column":"userprincipalname"},"userPrincipalName_MD5":{"column":"userprincipalname"},"product":{"value":"OneDrive"},"last_usage":{"column":"oneDriveLastActivityDate"},"installation_date":{"column":"oneDriveLicenseAssignDate"},"hasonedrivelicense":{"column":"hasonedrivelicense"}},"id":61034,"type":"map","name":"Transform O365 user activity data (OneDrive)","source":[{"step":61031}]},{"conditions":[{"$sql":"[hasonedrivelicense] == 'true'"}],"top":0,"id":61035,"type":"filter","name":"Find user activity data with an OneDrive license","source":{"step":61034}},{"mapRemaining":false,"columns":{"source":{"value":"O365_ua"},"source_table":{"column":"source_table"},"__source_id":{"column":"__source_id"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_Office365Data_user_activity] WHERE __source_id =' || __source_id"},"userprincipalname":{"column":"userprincipalname"},"userPrincipalName_MD5":{"column":"userprincipalname"},"product":{"value":"SharePoint"},"last_usage":{"column":"sharePointLastActivityDate"},"installation_date":{"column":"sharePointLicenseAssignDate"},"hassharepointlicense":{"column":"hassharepointlicense"}},"id":61036,"type":"map","name":"Transform O365 user activity data (SharePoint)","source":[{"step":61031}]},{"conditions":[{"$sql":"[hassharepointlicense] == 'true'"}],"top":0,"id":61037,"type":"filter","name":"Find user activity data with a SharePoint license","source":{"step":61036}},{"mapRemaining":false,"columns":{"source":{"value":"O365_ua"},"source_table":{"column":"source_table"},"__source_id":{"column":"__source_id"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_Office365Data_user_activity] WHERE __source_id =' || __source_id"},"userprincipalname":{"column":"userprincipalname"},"userPrincipalName_MD5":{"column":"userprincipalname"},"product":{"value":"Skype for Business"},"last_usage":{"column":"skypeForBusinessLastActivityDate"},"installation_date":{"column":"skypeForBusinessLicenseAssignDate"},"hasSkypeForBusinessLicense":{"column":"hasSkypeForBusinessLicense"}},"id":61038,"type":"map","name":"Transform O365 user activity data (Skype for Business)","source":[{"step":61031}]},{"conditions":[{"$sql":"[hasSkypeForBusinessLicense] == 'true'"}],"top":0,"id":61040,"type":"filter","name":"Find user activity data with a Skype for Business license","source":{"step":61038}},{"mapRemaining":false,"columns":{"source":{"value":"O365_ua"},"source_table":{"column":"source_table"},"__source_id":{"column":"__source_id"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_Office365Data_user_activity] WHERE __source_id =' || __source_id"},"userprincipalname":{"column":"userprincipalname"},"userPrincipalName_MD5":{"column":"userprincipalname"},"product":{"value":"Yammer"},"last_usage":{"column":"yammerLastActivityDate"},"installation_date":{"column":"yammerLicenseAssignDate"},"hasyammerlicense":{"column":"hasyammerlicense"}},"id":61041,"type":"map","name":"Transform O365 user activity data (Yammer)","source":[{"step":61031}]},{"conditions":[{"$sql":"[hasyammerlicense] == 'true'"}],"top":0,"id":61042,"type":"filter","name":"Find user activity data with a Yammer license","source":{"step":61041}},{"mapRemaining":false,"columns":{"source":{"value":"O365_ua"},"source_table":{"column":"source_table"},"__source_id":{"column":"__source_id"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_Office365Data_user_activity] WHERE __source_id =' || __source_id"},"userprincipalname":{"column":"userprincipalname"},"userPrincipalName_MD5":{"column":"userprincipalname"},"product":{"value":"Teams"},"last_usage":{"column":"teamslastactivitydate"},"installation_date":{"column":"teamsLicenseAssignDate"},"hasteamslicense":{"column":"hasteamslicense"}},"id":61043,"type":"map","name":"Transform O365 user activity data (Teams)","source":[{"step":61031}]},{"conditions":[{"$sql":"[hasteamslicense] == 'true'"}],"top":0,"id":61044,"type":"filter","name":"Find user activity data with a Teams license","source":{"step":61043}},{"mapRemaining":true,"columns":{},"id":61050,"type":"map","name":"Combine O365 user activity data","source":[{"step":61033},{"step":61035},{"step":61037},{"step":61040},{"step":61042},{"step":61044}]},{"strategy":"inner","sources":[{"step":100201},{"step":61050}],"on":["UserPrincipalName"],"conflicts":{"source":{"aggregation":{"type":"concat","separator":" | "}},"__source_id":{"aggregation":{"type":"concat","separator":" | "}},"source_table":{"aggregation":{"type":"concat","separator":" | "}},"__source_select_string":{"aggregation":{"type":"concat","separator":" | "}}},"id":61051,"type":"join","name":"Combine O365 user and user activity data (UserPrincipalName)"},{"strategy":"inner","sources":[{"step":100201},{"step":61050}],"on":["userPrincipalName_MD5"],"conflicts":{"source":{"aggregation":{"type":"concat","separator":" | "}},"__source_id":{"aggregation":{"type":"concat","separator":" | "}},"source_table":{"aggregation":{"type":"concat","separator":" | "}},"__source_select_string":{"aggregation":{"type":"concat","separator":" | "}}},"id":61053,"type":"join","name":"Combine O365 user and user activity data (userPrincipalName_MD5)"},{"mapRemaining":false,"columns":{"source":{"type":"nvarchar(255)","column":"source"},"__source_id":{"type":"nvarchar(255)","column":"__source_id"},"source_table":{"type":"nvarchar(255)","column":"source_table"},"__source_select_string":{"type":"nvarchar(max)","column":"__source_select_string"},"import_id":{"sql":"MD5(LOWER(userprincipalname || product))"},"import_account_id":{"type":"nvarchar(255)","sql":"'O365_' || userPrincipalName "},"generic_key":{"type":"nvarchar(255)","column":"product"},"quantity":{"value":1},"import_data_source_id":{"type":"nvarchar(255)","value":"m365"},"publisher":{"type":"nvarchar(255)","value":"Microsoft"},"product":{"type":"nvarchar(255)","column":"product"},"description":{"type":"nvarchar(255)","value":"Microsoft 365 User Activity"},"installation_date":{"column":"installation_date"},"last_usage":{"sql":"substr(last_usage, 1, 10)"}},"id":61052,"type":"map","name":"Add metadata, transform and format O365 inv raw generic app usage data","source":[{"step":61051},{"step":61053}]},{"mapRemaining":true,"columns":{"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"},"createddatetime":{"sql":"!transformDate(substr(createddatetime, 1, 10))"}},"id":62000,"type":"map","name":"Add metadata for O365 user sign ins data","source":[{"table":"Office365Data*-sign_ins"}],"target":{"name":"__snapshot_Office365Data_sign_ins"}},{"by":["userId","appId"],"strategy":{"type":"maxvalue","column":"createddatetime"},"id":62001,"type":"deduplicate","name":"Find most recent created datet ime for O365 user signins data","source":{"step":62000}},{"mapRemaining":false,"columns":{"source":{"value":"O365_si"},"source_table":{"column":"source_table"},"import_id":{"sql":"MD5(LOWER(appId || userId))"},"import_account_id":{"type":"nvarchar(255)","column":"userid"},"generic_key":{"type":"nvarchar(255)","column":"appid"},"quantity":{"value":1},"import_data_source_id":{"type":"nvarchar(255)","value":"m365_signins"},"publisher":{"type":"nvarchar(255)","value":"Microsoft"},"product":{"type":"nvarchar(255)","column":"appdisplayname"},"description":{"type":"nvarchar(255)","value":"Microsoft 365 SignIn"},"last_usage":{"column":"createddatetime"},"action":{"value":"insert"}},"id":62201,"type":"map","name":"Transform O365 user sign ins data","source":[{"step":62001}]},{"mapRemaining":true,"columns":{"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":62300,"type":"map","name":"Add and transform metadata for O365 user sign ins persistent data","source":[{"table":"*persistent_inv_raw_generic_Office365"}]},{"conditions":[{"$sql":"((julianday('now') - julianday(datetime(last_usage))) < 365)"}],"top":0,"id":62301,"type":"filter","name":"Filter O365 user sign ins persistent data","source":{"step":62300}},{"strategy":"outer","sources":[{"step":62201},{"step":62301}],"on":["import_id"],"conflicts":{"source":{"aggregation":{"type":"coalesce"}},"__timestamp":{"aggregation":{"type":"coalesce"}},"source_table":{"aggregation":{"type":"concat","separator":" | "}},"import_data_source_id":{"aggregation":{"type":"coalesce"}},"import_account_id":{"aggregation":{"type":"coalesce"}},"generic_key":{"aggregation":{"type":"coalesce"}},"quantity":{"aggregation":{"type":"coalesce"}},"publisher":{"aggregation":{"type":"coalesce"}},"product":{"aggregation":{"type":"coalesce"}},"description":{"aggregation":{"type":"coalesce"}},"last_usage":{"aggregation":{"type":"max"}},"action":{"aggregation":{"type":"coalesce"},"priority":[{"step":62301},{"step":62201}]}},"id":62400,"type":"join","name":"Combine new and persistent O365 user sign ins data"},{"mapRemaining":true,"columns":{"action":{"sql":"CASE WHEN !IsSingleTable(source_table) THEN [action] ELSE 'update' END"}},"id":62401,"type":"map","name":"Add and transform metadata for O365 user sign ins persistent data","source":[{"step":62400}],"target":{"name":"persistent_inv_raw_generic_Office365"}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":60000,"type":"map","name":"Add metadata for SCCM software ARP data (SQL)","source":[{"table":"SCCMSQLSoftwareARP*"}],"target":{"name":"__snapshot_SCCMSQLSoftwareARP"}},{"mapRemaining":true,"columns":{"source":{"value":"SCCM_SQL"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_SCCMSQLSoftwareARP] WHERE __source_id =' || __source_id"},"import_data_source_id":{"value":"SCCM_SQL"},"device_key":{"column":"import_device_id"}},"id":60001,"type":"map","name":"Transform SCCM software ARP data (SQL)","source":[{"step":60000}]},{"column":"import_device_id","lookup":[{"in":{"step":2999},"key":{"primary":"device_key","foreign":"device_key"},"take":{"column":"import_id"}}],"id":60002,"type":"enrich","name":"Enrich import device id for SCCM software ARP data (SQL)","source":{"step":60001}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":60003,"type":"map","name":"Add metadata for RV software ARP data","source":[{"table":"InventorySoftwareARP*"}],"target":{"name":"__snapshot_InventorySoftwareARP"}},{"mapRemaining":true,"columns":{"source":{"value":"RV"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_InventorySoftwareARP] WHERE __source_id =' || __source_id"},"import_data_source_id":{"value":"RV"},"device_key":{"column":"import_device_id"}},"id":60004,"type":"map","name":"Transform RV software ARP data","source":[{"step":60003}]},{"column":"import_device_id","lookup":[{"in":{"step":2999},"key":{"primary":"device_key","foreign":"device_key"},"take":{"column":"import_id"}}],"id":60005,"type":"enrich","name":"Enrich import device id for RV software ARP data","source":{"step":60004}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":60006,"type":"map","name":"Add metadata for SCCM software data (WMI)","source":[{"table":"SCCM_WMI*-Software"}],"target":{"name":"__snapshot_SCCM_WMI_Software"}},{"mapRemaining":true,"columns":{"source":{"value":"SCCM_WMI"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_SCCM_WMI_Software] WHERE __source_id =' || __source_id"},"import_data_source_id":{"value":"SCCM_WMI"},"device_key":{"column":"import_device_id"},"import_id":{"column":"id"},"installation_date":{"sql":"!transformDate(installation_date)"}},"id":60007,"type":"map","name":"Transform SCCM software data (WMI)","source":[{"step":60006}]},{"column":"import_device_id","lookup":[{"in":{"step":2999},"key":{"primary":"device_key","foreign":"device_key"},"take":{"column":"import_id"}}],"id":60008,"type":"enrich","name":"Enrich import device id for SCCM software data (WMI)","source":{"step":60007}},{"mapRemaining":true,"columns":{},"id":60009,"type":"map","name":"Combine software data","source":[{"step":60002},{"step":60005},{"step":60008}]},{"conditions":[{"$sql":"[import_device_id] IS NOT NULL"}],"top":0,"id":69997,"type":"filter","name":"Filter out software with missing information","source":{"step":60009}},{"by":["import_device_id","publisher","product","product_version"],"strategy":{"type":"any"},"id":69998,"type":"deduplicate","name":"Deduplicate software data","source":{"step":69997}},{"mapRemaining":false,"columns":{"source":{"type":"nvarchar(255)","column":"source"},"__source_id":{"type":"nvarchar(255)","column":"__source_id"},"source_table":{"type":"nvarchar(255)","column":"source_table"},"__source_select_string":{"type":"nvarchar(max)","column":"__source_select_string"},"import_id":{"type":"nvarchar(255)","column":"import_id"},"import_data_source_id":{"type":"nvarchar(255)","column":"import_data_source_id"},"publisher":{"type":"nvarchar(255)","column":"publisher","compact":true},"product":{"type":"nvarchar(255)","column":"product","compact":true},"product_version":{"type":"nvarchar(255)","column":"product_version","compact":true},"description":{"type":"nvarchar(255)","fallback":{"value":null},"column":"description"},"import_device_id":{"type":"nvarchar(255)","column":"import_device_id"},"installation_date":{"type":"nvarchar(255)","column":"installation_date"}},"id":69999,"type":"map","name":"Format software data","source":[{"step":69998}],"target":{"name":"result_inv_raw_arp"}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":40000,"type":"map","name":"Add metadata for SCCM file data (SQL)","source":[{"table":"SCCMSQLSoftwareFile"}],"target":{"name":"__snapshot_SCCMSQLSoftwareFile"}},{"mapRemaining":true,"columns":{"source":{"value":"SCCM_SQL"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_SCCMSQLSoftwareFile] WHERE __source_id =' || __source_id"},"import_data_source_id":{"value":"SCCM_SQL"},"device_key":{"column":"import_device_id"}},"id":40001,"type":"map","name":"Transform SCCM file data (SQL)","source":[{"step":40000}]},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":41000,"type":"map","name":"Add metadata for SCCM metering file data (SQL)","source":[{"table":"SCCMSQLSoftwareFileMetering*"}],"target":{"name":"__snapshot_SCCMSQLSoftwareFileMetering"}},{"mapRemaining":false,"columns":{"source":{"value":"SCCM"},"import_id":{"column":"import_id"},"last_usage":{"sql":"!transformDate(last_usage)"}},"id":41001,"type":"map","name":"Transform SCCM metering file data (SQL)","source":[{"step":41000}]},{"strategy":"left","sources":[{"step":40001},{"step":41001}],"on":["import_id"],"conflicts":{},"id":41002,"type":"join","name":"Combine SCCM file and metering information (SQL)"},{"column":"import_device_id","lookup":[{"in":{"step":2999},"key":{"primary":"device_key","foreign":"device_key"},"take":{"column":"import_id"}}],"id":40002,"type":"enrich","name":"Enrich import device id for SCCM file data (SQL)","source":{"step":41002}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":40003,"type":"map","name":"Add metadata for RV file data","source":[{"table":"InventorySoftwareFile*"}],"target":{"name":"__snapshot_InventorySoftwareFile"}},{"mapRemaining":true,"columns":{"source":{"value":"RV"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_InventorySoftwareFile] WHERE __source_id =' || __source_id"},"import_data_source_id":{"value":"RV"},"device_key":{"column":"import_device_id"},"last_usage":{"sql":"!transformDate(last_usage)"}},"id":40004,"type":"map","name":"Transform RV file data","source":[{"step":40003}]},{"column":"import_device_id","lookup":[{"in":{"step":2999},"key":{"primary":"device_key","foreign":"device_key"},"take":{"column":"import_id"}}],"id":40005,"type":"enrich","name":"Enrich import device id for RV file data","source":{"step":40004}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":40103,"type":"map","name":"Add metadata for SCCM metering file data (WMI)","source":[{"table":"SCCM_WMI*-SoftwareFileMetering"}],"target":{"name":"__snapshot_SCCM_WMI_SoftwareFileMetering"}},{"mapRemaining":true,"columns":{"source":{"value":"SCCM_WMI"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_SCCM_WMI_SoftwareFileMetering] WHERE __source_id =' || __source_id"},"import_data_source_id":{"value":"SCCM_WMI"},"import_id":{"column":"id"},"description":{"column":"file_description"},"device_key":{"column":"import_device_id"},"user_key":{"column":"import_user_id"},"last_usage":{"sql":"!transformDate(last_usage)"}},"id":40104,"type":"map","name":"Transform SCCM metering file data (WMI)","source":[{"step":40103}]},{"column":"import_device_id","lookup":[{"in":{"step":2999},"key":{"primary":"device_key","foreign":"device_key"},"take":{"column":"import_id"}},{"take":{"column":"device_key"}}],"id":40105,"type":"enrich","name":"Enrich import device id for SCCM metering file data (WMI)","source":{"step":40104}},{"column":"import_account_id","lookup":[{"in":{"step":2002998},"key":{"primary":"user_key","foreign":"user_key"},"take":{"column":"import_id"}},{"take":{"column":"user_key"}}],"id":40106,"type":"enrich","name":"Enrich import account id for SCCM metering file data (WMI)","source":{"step":40105}},{"mapRemaining":false,"columns":{"source":{"type":"nvarchar(255)","column":"source"},"__source_id":{"type":"nvarchar(255)","column":"__source_id"},"source_table":{"type":"nvarchar(255)","column":"source_table"},"__source_select_string":{"type":"nvarchar(max)","column":"__source_select_string"},"import_id":{"type":"nvarchar(255)","column":"import_id"},"import_data_source_id":{"type":"nvarchar(255)","column":"import_data_source_id"},"publisher":{"type":"nvarchar(255)","column":"publisher"},"product":{"type":"nvarchar(255)","column":"product"},"product_version":{"type":"nvarchar(255)","column":"product_version"},"file_name":{"type":"nvarchar(255)","column":"file_name"},"file_path":{"type":"nvarchar(255)","column":"file_path"},"last_usage":{"type":"nvarchar(255)","fallback":{"value":null},"column":"last_usage"},"usage_frequency_month":{"type":"INT","fallback":{"value":null},"column":"usage_frequency_month"},"usage_time":{"type":"INT","fallback":{"value":null},"column":"usage_time"},"description":{"type":"nvarchar(255)","column":"description"},"import_device_id":{"type":"nvarchar(255)","column":"import_device_id"},"import_account_id":{"type":"nvarchar(255)","fallback":{"value":null},"column":"import_account_id"},"installation_date":{"type":"nvarchar(255)","column":"installation_date"}},"id":40006,"type":"map","name":"Combine file data","source":[{"step":40002},{"step":40005},{"step":40106}]},{"conditions":[{"$sql":"[import_device_id] IS NOT NULL"}],"top":0,"id":49997,"type":"filter","name":"Filter out file with missing information","source":{"step":40006}},{"by":["import_device_id","import_account_id","publisher","product","product_version","file_name","file_path"],"strategy":{"type":"maxvalue","column":"last_usage"},"id":49998,"type":"deduplicate","name":"Find most recent last usage for file data","source":{"step":49997}},{"mapRemaining":false,"columns":{"source":{"type":"nvarchar(255)","column":"source"},"__source_id":{"type":"nvarchar(255)","column":"__source_id"},"source_table":{"type":"nvarchar(255)","column":"source_table"},"__source_select_string":{"type":"nvarchar(max)","column":"__source_select_string"},"import_id":{"type":"nvarchar(255)","column":"import_id"},"import_data_source_id":{"type":"nvarchar(255)","column":"import_data_source_id"},"publisher":{"type":"nvarchar(255)","column":"publisher","compact":true},"product":{"type":"nvarchar(255)","column":"product","compact":true},"product_version":{"type":"nvarchar(255)","column":"product_version","compact":true},"description":{"type":"nvarchar(255)","column":"description"},"file_name":{"type":"nvarchar(255)","column":"file_name"},"file_path":{"type":"nvarchar(255)","column":"file_path"},"last_usage":{"type":"nvarchar(255)","column":"last_usage"},"usage_frequency_month":{"type":"INT","column":"usage_frequency_month"},"usage_time":{"type":"INT","column":"usage_time"},"import_device_id":{"type":"nvarchar(255)","column":"import_device_id"},"import_account_id":{"type":"nvarchar(255)","fallback":{"value":null},"column":"import_account_id"},"installation_date":{"type":"nvarchar(255)","column":"installation_date"}},"id":49999,"type":"map","name":"Format file data","source":[{"step":49998}],"target":{"name":"result_inv_raw_file"}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":50000,"type":"map","name":"Add metadata for SCCM OS data (SQL)","source":[{"table":"SCCMSQLSoftwareOS*"}],"target":{"name":"__snapshot_SCCMSQLSoftwareOS"}},{"mapRemaining":true,"columns":{"source":{"value":"SCCM_SQL"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_SCCMSQLSoftwareOS] WHERE __source_id =' || __source_id"},"import_data_source_id":{"value":"SCCM_SQL"},"device_key":{"column":"import_device_id"}},"id":50001,"type":"map","name":"Transform SCCM OS data (SQL)","source":[{"step":50000}]},{"column":"import_device_id","lookup":[{"in":{"step":2999},"key":{"primary":"device_key","foreign":"device_key"},"take":{"column":"import_id"}}],"id":50002,"type":"enrich","name":"Enrich import device id for SCCM OS data (SQL)","source":{"step":50001}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":50003,"type":"map","name":"Add metadata for RV OS data","source":[{"table":"InventorySoftwareOS*"}],"target":{"name":"__snapshot_InventorySoftwareOS"}},{"mapRemaining":true,"columns":{"source":{"value":"RV"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_InventorySoftwareOS] WHERE __source_id =' || __source_id"},"import_data_source_id":{"value":"RV"},"device_key":{"column":"import_device_id"},"installation_date":{"sql":"!transformDate(installation_date)"}},"id":50004,"type":"map","name":"Transform RV OS data","source":[{"step":50003}]},{"column":"import_device_id","lookup":[{"in":{"step":2999},"key":{"primary":"device_key","foreign":"device_key"},"take":{"column":"import_id"}}],"id":50005,"type":"enrich","name":"Enrich import device id for RV OS data","source":{"step":50004}},{"mapRemaining":false,"columns":{"source":{"type":"nvarchar(255)","column":"source"},"__source_id":{"type":"nvarchar(255)","column":"__source_id"},"source_table":{"type":"nvarchar(255)","column":"source_table"},"__source_select_string":{"type":"nvarchar(max)","column":"__source_select_string"},"import_id":{"type":"nvarchar(255)","column":"import_id"},"import_data_source_id":{"type":"nvarchar(255)","column":"import_data_source_id"},"generic_key":{"type":"nvarchar(255)","column":"generic_key"},"publisher":{"type":"nvarchar(255)","column":"publisher"},"product":{"type":"nvarchar(255)","column":"product"},"product_version":{"type":"nvarchar(255)","column":"product_version"},"description":{"type":"nvarchar(255)","fallback":{"value":null},"column":"description"},"quantity":{"type":"nvarchar(255)","column":"quantity"},"import_device_id":{"type":"nvarchar(255)","fallback":{"value":null},"column":"import_device_id"},"import_account_id":{"type":"nvarchar(255)","fallback":{"value":null},"column":"import_account_id"},"installation_date":{"type":"nvarchar(255)","column":"installation_date"}},"id":50010,"type":"map","name":"Combine and format OS data","source":[{"step":50002},{"step":50005}]},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":50011,"type":"map","name":"Add metadata for RV other SW data","source":[{"table":"InventorySoftwareOthersw*"}],"target":{"name":"__snapshot_InventorySoftwareOthersw"}},{"mapRemaining":true,"columns":{"source":{"value":"RV"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_InventorySoftwareOthersw] WHERE __source_id =' || __source_id"},"import_data_source_id":{"value":"RV"},"device_key":{"column":"import_device_id"},"installation_date":{"sql":"!transformDate(installation_date)"}},"id":50012,"type":"map","name":"Transform RV other SW data","source":[{"step":50011}]},{"column":"import_device_id","lookup":[{"in":{"step":2999},"key":{"primary":"device_key","foreign":"device_key"},"take":{"column":"import_id"}}],"id":50013,"type":"enrich","name":"Enrich import device id for RV other SW data","source":{"step":50012}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":50021,"type":"map","name":"Add metadata for RV Oracle data","source":[{"table":"InventorySoftwareOracle*"}],"target":{"name":"__snapshot_InventorySoftwareOracle"}},{"mapRemaining":true,"columns":{"source":{"value":"RV"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_InventorySoftwareOracle] WHERE __source_id =' || __source_id"},"import_data_source_id":{"value":"RV"},"device_key":{"column":"import_device_id"},"installation_date":{"sql":"!transformDate(installation_date)"},"last_usage":{"sql":"!transformDate(last_usage)"},"first_usage":{"sql":"!transformDate(first_usage)"}},"id":50022,"type":"map","name":"Transform RV Oracle data","source":[{"step":50021}]},{"column":"import_device_id","lookup":[{"in":{"step":2999},"key":{"primary":"device_key","foreign":"device_key"},"take":{"column":"import_id"}}],"id":50023,"type":"enrich","name":"Enrich import device id for RV Oracle data","source":{"step":50022}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":50031,"type":"map","name":"Add metadata for RV Microsoft data","source":[{"table":"InventorySoftwareMicrosoft*"}],"target":{"name":"__snapshot_InventorySoftwareMicrosoft"}},{"mapRemaining":true,"columns":{"source":{"value":"RV"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_InventorySoftwareMicrosoft] WHERE __source_id =' || __source_id"},"import_data_source_id":{"value":"RV"},"device_key":{"column":"import_device_id"},"installation_date":{"sql":"!transformDate(installation_date)"}},"id":50032,"type":"map","name":"Transform RV Microsoft data","source":[{"step":50031}]},{"column":"import_device_id","lookup":[{"in":{"step":2999},"key":{"primary":"device_key","foreign":"device_key"},"take":{"column":"import_id"}}],"id":50033,"type":"enrich","name":"Enrich import device id for RV Microsoft data","source":{"step":50032}},{"mapRemaining":true,"columns":{},"id":50034,"type":"map","name":"Combine generic software data","source":[{"step":50010},{"step":50013},{"step":50023},{"step":50033},{"step":10039998},{"step":61008},{"step":61029},{"step":61052},{"step":62401}]},{"conditions":[{"$sql":"[import_device_id] IS NOT NULL OR [import_account_id] IS NOT NULL"}],"top":0,"id":50098,"type":"filter","name":"Filter out generic software data with missing information","source":{"step":50034}},{"by":["import_device_id","import_account_id","publisher","product","product_version","instance_name"],"strategy":"any","id":50099,"type":"deduplicate","name":"Deduplicate generic software data","source":{"step":50098}},{"mapRemaining":false,"columns":{"source":{"column":"source","type":"nvarchar(255)"},"__source_id":{"column":"__source_id","type":"nvarchar(255)"},"source_table":{"column":"source_table","type":"nvarchar(255)"},"__source_select_string":{"column":"__source_select_string","type":"nvarchar(max)"},"import_id":{"column":"import_id","type":"nvarchar(255)"},"import_data_source_id":{"column":"import_data_source_id","type":"nvarchar(255)"},"generic_key":{"column":"generic_key","type":"nvarchar(255)"},"publisher":{"column":"publisher","compact":true,"type":"nvarchar(255)"},"product":{"column":"product","compact":true,"type":"nvarchar(255)"},"product_version":{"column":"product_version","compact":true,"type":"nvarchar(255)"},"description":{"column":"description","compact":true,"type":"nvarchar(255)"},"quantity":{"column":"quantity","type":"nvarchar(255)"},"action":{"column":"action","fallback":{"value":null},"type":"nvarchar(255)"},"last_usage":{"column":"last_usage","fallback":{"value":null},"type":"nvarchar(255)"},"installation_date":{"column":"installation_date","fallback":{"value":null},"type":"nvarchar(255)"},"import_device_id":{"column":"import_device_id","compact":true,"type":"nvarchar(255)"},"import_account_id":{"column":"import_account_id","compact":true,"type":"nvarchar(255)"},"instance_name":{"column":"instance_name"}},"id":50100,"type":"map","name":"Format generic software data","source":[{"step":50099}],"target":{"name":"result_inv_raw_generic"}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":50131,"type":"map","name":"Add metadata for RV MSI data","source":[{"table":"InventorySoftwareMSI*"}],"target":{"name":"__snapshot_InventorySoftwareMSI"}},{"mapRemaining":true,"columns":{"source":{"value":"RV"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_InventorySoftwareMSI] WHERE __source_id =' || __source_id"},"import_data_source_id":{"value":"RV"},"device_key":{"column":"import_device_id"}},"id":50132,"type":"map","name":"Transform RV MSI data","source":[{"step":50131}]},{"column":"import_device_id","lookup":[{"in":{"step":2999},"key":{"primary":"device_key","foreign":"device_key"},"take":{"column":"import_id"}}],"id":50133,"type":"enrich","name":"Enrich import device id for RV MSI data","source":{"step":50132}},{"conditions":[{"$sql":"[import_device_id] IS NOT NULL"}],"top":0,"id":50134,"type":"filter","name":"Filter out RV MSI with missing information","source":{"step":50133}},{"by":["import_device_id","publisher","product","product_version"],"strategy":{"type":"any"},"id":50135,"type":"deduplicate","name":"Deduplicate RV MSI data","source":{"step":50134}},{"mapRemaining":false,"columns":{"source":{"type":"nvarchar(255)","column":"source"},"__source_id":{"type":"nvarchar(255)","column":"__source_id"},"source_table":{"type":"nvarchar(255)","column":"source_table"},"__source_select_string":{"type":"nvarchar(max)","column":"__source_select_string"},"import_id":{"type":"nvarchar(255)","column":"import_id"},"import_data_source_id":{"type":"nvarchar(255)","column":"import_data_source_id"},"publisher":{"type":"nvarchar(255)","column":"publisher","compact":true},"product":{"type":"nvarchar(255)","column":"product","compact":true},"product_version":{"type":"nvarchar(255)","column":"product_version","compact":true},"import_device_id":{"type":"nvarchar(255)","column":"import_device_id"},"product_code":{"type":"nvarchar(255)","column":"guid"},"installation_date":{"type":"nvarchar(255)","column":"installation_date"}},"id":50199,"type":"map","name":"Format RV MSI data","source":[{"step":50135}],"target":{"name":"result_inv_raw_msi"}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":30000,"type":"map","name":"Add metadata for SCCM tag data (SQL)","source":[{"table":"SCCMSQLSoftwareTag*"}],"target":{"name":"__snapshot_SCCMSQLSoftwareTag"}},{"mapRemaining":true,"columns":{"source":{"value":"SCCM_SQL"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_SCCMSQLSoftwareTag] WHERE __source_id =' || __source_id"},"import_data_source_id":{"value":"SCCM_SQL"},"device_key":{"column":"import_device_id"}},"id":30001,"type":"map","name":"Transform SCCM tag data (SQL)","source":[{"step":30000}]},{"column":"import_device_id","lookup":[{"in":{"step":2999},"key":{"primary":"device_key","foreign":"device_key"},"take":{"column":"import_id"}}],"id":30002,"type":"enrich","name":"Enrich import device id for SCCM tag data (SQL)","source":{"step":30001}},{"mapRemaining":true,"columns":{"__source_id":{"auto":"Number"},"source_table":{"auto":"TableName"},"__timestamp":{"auto":"Date"}},"id":30003,"type":"map","name":"Add metadata for RV tag data","source":[{"table":"InventorySoftwareTag*"}],"target":{"name":"__snapshot_InventorySoftwareTag"}},{"mapRemaining":true,"columns":{"source":{"value":"RV"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_InventorySoftwareTag] WHERE __source_id =' || __source_id"},"import_data_source_id":{"value":"RV"},"device_key":{"column":"import_device_id"}},"id":30004,"type":"map","name":"Transform RV tag data","source":[{"step":30003}]},{"column":"import_device_id","lookup":[{"in":{"step":2999},"key":{"primary":"device_key","foreign":"device_key"},"take":{"column":"import_id"}}],"id":30005,"type":"enrich","name":"Enrich import device id for RV tag data","source":{"step":30004}},{"mapRemaining":true,"columns":{},"id":30010,"type":"map","name":"Combine tag data","source":[{"step":30002},{"step":30005}]},{"conditions":[{"$sql":"[import_device_id] IS NOT NULL"}],"top":0,"id":39997,"type":"filter","name":"Filter out tags with missing information","source":{"step":30010}},{"by":["import_device_id","manufacturer_name","product_title","product_ver_name"],"strategy":{"type":"any"},"id":39998,"type":"deduplicate","name":"Deduplicate tag data","source":{"step":39997}},{"mapRemaining":false,"columns":{"source":{"type":"nvarchar(255)","column":"source"},"__source_id":{"type":"nvarchar(255)","column":"__source_id"},"source_table":{"type":"nvarchar(255)","column":"source_table"},"__source_select_string":{"type":"nvarchar(max)","column":"__source_select_string"},"import_id":{"type":"nvarchar(255)","column":"import_id"},"import_data_source_id":{"type":"nvarchar(255)","column":"import_data_source_id"},"publisher":{"type":"nvarchar(255)","column":"manufacturer_name","compact":true},"product":{"type":"nvarchar(255)","column":"product_title","compact":true},"product_version":{"type":"nvarchar(255)","column":"product_ver_name","compact":true},"description":{"type":"nvarchar(255)","column":"description"},"import_device_id":{"type":"nvarchar(255)","column":"import_device_id"},"installation_date":{"type":"nvarchar(255)","column":"installation_date"}},"id":39999,"type":"map","name":"Format tag data","source":[{"step":39998}],"target":{"name":"result_inv_raw_tag"}},{"mapRemaining":false,"columns":{"source":{"type":"nvarchar(255)","column":"source"},"__source_id":{"type":"nvarchar(255)","column":"__source_id"},"source_table":{"type":"nvarchar(255)","column":"source_table"},"__source_select_string":{"type":"nvarchar(max)","column":"__source_select_string"},"import_id":{"type":"nvarchar(255)","column":"import_id"},"import_data_source_id":{"type":"nvarchar(255)","column":"import_data_source_id"},"publisher":{"type":"nvarchar(255)","column":"publisher"},"product":{"type":"nvarchar(255)","column":"product"},"product_version":{"type":"nvarchar(255)","column":"product_version"},"description":{"type":"nvarchar(255)","column":"description"},"product_code":{"type":"nvarchar(255)","fallback":{"value":null},"column":"product_code"},"file_name":{"type":"nvarchar(255)","fallback":{"value":null},"column":"file_name"},"file_path":{"type":"nvarchar(255)","fallback":{"value":null},"column":"file_path"},"last_usage":{"type":"nvarchar(255)","fallback":{"value":null},"column":"last_usage"},"usage_frequency_month":{"type":"INT","fallback":{"value":null},"column":"usage_frequency_month"},"usage_time":{"type":"INT","fallback":{"value":null},"column":"usage_time"},"installation_date":{"type":"nvarchar(255)","fallback":{"value":null},"column":"installation_date"},"import_device_id":{"type":"nvarchar(255)","fallback":{"value":null},"column":"import_device_id"},"import_account_id":{"type":"nvarchar(255)","fallback":{"value":null},"column":"import_account_id"}},"id":10500,"type":"map","name":"Combine and format software data","source":[{"step":69999},{"step":49999},{"step":50100},{"step":50199},{"step":39999}]},{"by":["source","import_device_id","import_account_id","publisher","product","product_version","file_name","file_path"],"action":{"type":"deduplicate","groupCountColumn":"sourceCount","columns":{"source_table":{"type":"concat","separator":" | "},"__source_id":{"type":"concat","separator":" | "},"__source_select_string":{"type":"concat","separator":" | "}}},"id":10501,"type":"group","name":"Group software data","source":{"step":10500}},{"by":["import_device_id","import_account_id","publisher","product","product_version","file_name","file_path"],"action":{"type":"deduplicate","columns":{"source":{"type":"concat","separator":" | "},"source_table":{"type":"concat","separator":" | "},"__source_id":{"type":"concat","separator":" | "},"__source_select_string":{"type":"concat","separator":" | "}}},"id":10511,"type":"group","name":"Group software data","source":{"step":10501}},{"mapRemaining":false,"columns":{"source":{"type":"nvarchar(255)","column":"source"},"__source_id":{"type":"nvarchar(255)","column":"__source_id"},"source_table":{"type":"nvarchar(510)","column":"source_table"},"__source_select_string":{"type":"nvarchar(max)","column":"__source_select_string"},"import_data_source_id":{"type":"nvarchar(255)","value":"DataHub"},"import_device_id":{"type":"nvarchar(255)","column":"import_device_id"},"import_account_id":{"type":"nvarchar(255)","column":"import_account_id"},"publisher":{"type":"nvarchar(255)","column":"publisher"},"product":{"type":"nvarchar(255)","column":"product"},"product_version":{"type":"nvarchar(255)","column":"product_version"},"description":{"type":"nvarchar(255)","sql":"description"},"product_code":{"type":"nvarchar(255)","sql":"product_code"},"file_name":{"type":"nvarchar(255)","column":"file_name"},"file_path":{"type":"nvarchar(255)","column":"file_path"},"last_usage":{"type":"nvarchar(255)","sql":"last_usage"},"usage_frequency_month":{"type":"INT","column":"usage_frequency_month"},"usage_time":{"type":"INT","column":"usage_time"},"installation_date":{"type":"nvarchar(255)","sql":"installation_date"}},"id":10502,"type":"map","name":"Format software data","source":[{"step":10511}],"target":{"name":"result_software"}},{"mapRemaining":true,"columns":{"source":{"type":"nvarchar(255)","value":null},"__source_id":{"type":"nvarchar(255)","value":null},"source_table":{"type":"nvarchar(510)","value":null},"__source_select_string":{"type":"nvarchar(max)","value":null}},"id":11502,"type":"map","name":"Format software data to reset source information","source":[{"step":10502}]},{"by":["source","publisher","product","product_version","file_name","file_path"],"action":{"type":"deduplicate","groupCountColumn":"softwareSourceCount","columns":{"source_table":{"type":"max"}}},"id":10540,"type":"group","name":"Group software data","source":{"step":10502}},{"by":["publisher","product","product_version","file_name","file_path"],"action":{"type":"deduplicate","groupCountColumn":"softwareCount","columns":{"source":{"type":"concat","separator":" | "},"source_table":{"type":"concat","separator":" | "}}},"id":10640,"type":"group","name":"Group software data","source":{"step":10540}},{"mapRemaining":false,"columns":{"source":{"type":"nvarchar(255)","column":"source"},"source_table":{"type":"nvarchar(510)","column":"source_table"},"InstalledOn":{"type":"INT","value":0},"UsedBy":{"type":"INT","value":0},"import_data_source_id":{"type":"nvarchar(255)","column":"import_data_source_id"},"publisher":{"type":"nvarchar(255)","column":"publisher"},"product":{"type":"nvarchar(255)","column":"product"},"product_version":{"type":"nvarchar(255)","column":"product_version"},"description":{"type":"nvarchar(255)","column":"description"},"product_code":{"type":"nvarchar(255)","column":"product_code"},"file_name":{"type":"nvarchar(255)","column":"file_name"},"file_path":{"type":"nvarchar(255)","column":"file_path"}},"id":10541,"type":"map","name":"Format default software count data","source":[{"step":10640}]},{"by":["import_device_id","publisher","product","product_version","file_name","file_path"],"action":{"type":"deduplicate","columns":{}},"id":10503,"type":"group","name":"Determine software data for devices","source":{"step":11502}},{"conditions":[{"$sql":"[import_device_id] IS NOT NULL"}],"top":0,"id":10504,"type":"filter","name":"Filter out software with non device information","source":{"step":10503}},{"by":["publisher","product","product_version","file_name","file_path"],"action":{"type":"deduplicate","groupCountColumn":"InstalledOn","columns":{}},"id":10505,"type":"group","name":"Count software data for devices","source":{"step":10504}},{"by":["import_account_id","publisher","product","product_version","file_name","file_path"],"action":{"type":"deduplicate","columns":{}},"id":10506,"type":"group","name":"Determine software data for users","source":{"step":11502}},{"conditions":[{"$sql":"[import_account_id] IS NOT NULL"}],"top":0,"id":10507,"type":"filter","name":"Filter out software with non user information","source":{"step":10506}},{"by":["publisher","product","product_version","file_name","file_path"],"action":{"type":"deduplicate","groupCountColumn":"UsedBy","columns":{}},"id":10508,"type":"group","name":"Count software data for users","source":{"step":10507}},{"mapRemaining":true,"columns":{},"id":10550,"type":"map","name":"Combine counts for software data","source":[{"step":10505},{"step":10508},{"step":10541}]},{"by":["publisher","product","product_version","file_name","file_path"],"action":{"type":"deduplicate","groupCountColumn":"DeviceUserMatch","columns":{"import_device_id":{"type":"concat","separator":" | "},"import_account_id":{"type":"concat","separator":" | "},"InstalledOn":{"type":"max"},"UsedBy":{"type":"max"},"source":{"type":"max"},"source_table":{"type":"max"}}},"id":10551,"type":"group","name":"Group software count data","source":{"step":10550}},{"mapRemaining":false,"columns":{"source":{"type":"nvarchar(255)","column":"source"},"source_table":{"type":"nvarchar(510)","column":"source_table"},"InstalledOn":{"type":"INT","column":"InstalledOn"},"UsedBy":{"type":"INT","column":"UsedBy"},"import_data_source_id":{"type":"nvarchar(255)","column":"import_data_source_id"},"publisher":{"type":"nvarchar(255)","column":"publisher"},"product":{"type":"nvarchar(255)","column":"product"},"product_version":{"type":"nvarchar(255)","column":"product_version"},"file_name":{"type":"nvarchar(255)","column":"file_name"},"file_path":{"type":"nvarchar(255)","column":"file_path"},"description":{"type":"nvarchar(max)","sql":"description"},"product_code":{"type":"nvarchar(255)","sql":"product_code"}},"id":10555,"type":"map","name":"Format software summary data","source":[{"step":10551}],"target":{"name":"result_software_summary"}},{"mapRemaining":true,"columns":{"source":{"value":"SCCM_WMI"},"__source_id":{"column":"__source_id"},"source_table":{"column":"source_table"},"__source_select_string":{"sql":"'SELECT * FROM [DataTransformation-__snapshot_SCCM_WMI_UserDeviceRelations] WHERE __source_id =' || __source_id"},"import_data_source_id":{"value":"SCCM_WMI"},"device_key":{"column":"import_device_id"},"user_key":{"column":"import_user_id"},"import_id":{"column":"id"}},"id":881001,"type":"map","name":"Transform SCCM user device relation data (WMI)","source":[{"step":200100}]},{"column":"import_device_id","lookup":[{"in":{"step":2999},"key":{"primary":"device_key","foreign":"device_key"},"take":{"column":"import_id"}}],"id":881002,"type":"enrich","name":"Enrich import device id for SCCM metering file data (WMI)","source":{"step":881001}},{"column":"import_user_id","lookup":[{"in":{"step":2002998},"key":{"primary":"user_key","foreign":"user_key"},"take":{"column":"import_id"}}],"id":881003,"type":"enrich","name":"Enrich import user id for SCCM metering file data (WMI)","source":{"step":881002}},{"mapRemaining":false,"columns":{"source":{"type":"nvarchar(255)","column":"source"},"__source_id":{"type":"nvarchar(255)","column":"__source_id"},"source_table":{"type":"nvarchar(255)","column":"source_table"},"__source_select_string":{"type":"nvarchar(max)","column":"__source_select_string"},"import_id":{"type":"nvarchar(255)","column":"import_id"},"import_data_source_id":{"type":"nvarchar(255)","column":"import_data_source_id"},"import_user_id":{"type":"nvarchar(255)","column":"import_user_id"},"import_device_id":{"type":"nvarchar(255)","column":"import_device_id"},"creation_time":{"type":"nvarchar(255)","column":"creation_time"},"is_active":{"type":"bit","column":"is_active"},"is_primary":{"type":"bit","column":"is_active"}},"id":881004,"type":"map","name":"Format result user device relation data","source":[{"step":881003}],"target":{"name":"result_user_device_relations"}}]}