Good afternoon!
I’ve recently installed NetObserv 7.0.2 and certain enrichment data is unavailable. When I investigate further, I can see that none of the incoming fields contain data for these objects. Here is a screenshot for reference.
I’m unsure what the issue could be. My configuration is below:
EF_ACCOUNT_ID: "REMOVED"
#EF_FLOW_LICENSED_UNITS: ""
EF_FLOW_LICENSE_KEY: "REMOVED"
EF_LICENSE_ACCEPTED: "true"
#EF_API_BASIC_AUTH_ENABLE: "false"
#EF_API_BASIC_AUTH_PASSWORD: ""
#EF_API_BASIC_AUTH_USERNAME: ""
#EF_API_IP: 0.0.0.0
#EF_API_PORT: 8080
#EF_API_TLS_CERT_FILEPATH: ""
#EF_API_TLS_ENABLE: "false"
#EF_API_TLS_KEY_FILEPATH: ""
#EF_INSTANCE_NAME: default
#EF_LOGGER_DEVELOPMENT_ENABLE: "false"
#EF_LOGGER_ENCODING: console
#EF_LOGGER_FILE_LOG_COMPRESS: ""
#EF_LOGGER_FILE_LOG_ENABLE: "false"
#EF_LOGGER_FILE_LOG_FILENAME: /var/log/elastiflow/flowcoll/flowcoll.log
#EF_LOGGER_FILE_LOG_MAX_AGE: 0
#EF_LOGGER_FILE_LOG_MAX_BACKUPS: 4
#EF_LOGGER_FILE_LOG_MAX_SIZE: 100
#EF_LOGGER_LEVEL: info
#EF_INPUT_FLOW_BENCHMARK_ENABLE: "false"
#EF_INPUT_FLOW_BENCHMARK_PACKET_FILEPATH: /etc/elastiflow/benchmark/flow/packets.txt
EF_FLOW_SERVER_UDP_IP: 10.0.40.90
EF_FLOW_SERVER_UDP_PORT: 9995
#EF_FLOW_SERVER_UDP_READ_BUFFER_MAX_SIZE: 33554432
#AWS_ACCESS_KEY_ID: ""
#AWS_REGION: ""
#AWS_SECRET_ACCESS_KEY: ""
#EF_AWS_VPC_FLOW_LOG_S3_BUCKET: ""
#EF_AWS_VPC_FLOW_LOG_S3_ENABLE: "false"
#EF_AWS_VPC_FLOW_LOG_S3_POOL_SIZE: 1
#EF_AWS_VPC_FLOW_LOG_S3_PREFIX: AWSLogs
#EF_AWS_VPC_FLOW_LOG_S3_TLS_CA_CERT_FILEPATH: ""
#EF_AWS_VPC_FLOW_LOG_S3_TLS_ENABLE: "false"
#EF_AWS_VPC_FLOW_LOG_S3_TLS_MIN_VERSION: 1.2
#EF_AWS_VPC_FLOW_LOG_S3_TLS_SKIP_VERIFICATION: "false"
#EF_PROCESSOR_DECODE_IPFIX_ENABLE: "true"
#EF_PROCESSOR_DECODE_MAX_RECORDS_PER_PACKET: 64
#EF_PROCESSOR_DECODE_NETFLOW1_ENABLE: "true"
#EF_PROCESSOR_DECODE_NETFLOW5_ENABLE: "true"
#EF_PROCESSOR_DECODE_NETFLOW6_ENABLE: "true"
#EF_PROCESSOR_DECODE_NETFLOW7_ENABLE: "true"
#EF_PROCESSOR_DECODE_NETFLOW9_ENABLE: "true"
#EF_PROCESSOR_DECODE_SFLOW5_ENABLE: "true"
#EF_PROCESSOR_DECODE_SFLOW_COUNTERS_ENABLE: "true"
#EF_PROCESSOR_DECODE_SFLOW_FLOWS_ENABLE: "true"
#EF_PROCESSOR_DECODE_SFLOW_FLOWS_KEEP_SAMPLES: "false"
#EF_PROCESSOR_DROP_FIELDS: ""
#EF_PROCESSOR_DURATION_PRECISION: ms
#EF_PROCESSOR_ENRICH_APP_ID_ENABLE: "false"
#EF_PROCESSOR_ENRICH_APP_ID_PATH: ""
#EF_PROCESSOR_ENRICH_APP_ID_TTL: 7200
#EF_PROCESSOR_ENRICH_APP_IPPORT_ENABLE: "false"
#EF_PROCESSOR_ENRICH_APP_IPPORT_PATH: ""
#EF_PROCESSOR_ENRICH_APP_IPPORT_PRIVATE: "true"
#EF_PROCESSOR_ENRICH_APP_IPPORT_PUBLIC: "false"
#EF_PROCESSOR_ENRICH_APP_IPPORT_TTL: 7200
#EF_PROCESSOR_ENRICH_APP_REFRESH_RATE: 15
#EF_PROCESSOR_ENRICH_ASN_PREF: lookup
#EF_PROCESSOR_ENRICH_COMMUNITYID_ENABLE: "true"
#EF_PROCESSOR_ENRICH_COMMUNITYID_SEED: 0
#EF_PROCESSOR_ENRICH_CONVERSATIONID_ENABLE: "true"
#EF_PROCESSOR_ENRICH_CONVERSATIONID_SEED: 0
#EF_PROCESSOR_ENRICH_IPADDR_DNS_ENABLE: "false"
#EF_PROCESSOR_ENRICH_IPADDR_DNS_INCLEXCL_PATH: ""
#EF_PROCESSOR_ENRICH_IPADDR_DNS_INCLEXCL_REFRESH_RATE: 15
#EF_PROCESSOR_ENRICH_IPADDR_DNS_NAMESERVER_IP: ""
#EF_PROCESSOR_ENRICH_IPADDR_DNS_NAMESERVER_TIMEOUT: 3000
#EF_PROCESSOR_ENRICH_IPADDR_DNS_RESOLVE_PRIVATE: "true"
#EF_PROCESSOR_ENRICH_IPADDR_DNS_RESOLVE_PUBLIC: "true"
#EF_PROCESSOR_ENRICH_IPADDR_DNS_USERDEF_PATH: ""
#EF_PROCESSOR_ENRICH_IPADDR_DNS_USERDEF_REFRESH_RATE: 15
#EF_PROCESSOR_ENRICH_IPADDR_MAXMIND_ASN_ENABLE: "false"
#EF_PROCESSOR_ENRICH_IPADDR_MAXMIND_ASN_PATH: /etc/elastiflow/maxmind/GeoLite2-ASN.mmdb
EF_PROCESSOR_ENRICH_IPADDR_MAXMIND_GEOIP_ENABLE: "true"
#EF_PROCESSOR_ENRICH_IPADDR_MAXMIND_GEOIP_INCLEXCL_PATH: ""
#EF_PROCESSOR_ENRICH_IPADDR_MAXMIND_GEOIP_INCLEXCL_REFRESH_RATE: 15
#EF_PROCESSOR_ENRICH_IPADDR_MAXMIND_GEOIP_LANG: en
EF_PROCESSOR_ENRICH_IPADDR_MAXMIND_GEOIP_PATH: /usr/share/GeoIP/GeoLite2-City.mmdb
#EF_PROCESSOR_ENRICH_IPADDR_MAXMIND_GEOIP_VALUES: city,country,country_code,location,timezone
#EF_PROCESSOR_ENRICH_IPADDR_METADATA_ENABLE: "true"
#EF_PROCESSOR_ENRICH_IPADDR_METADATA_REFRESH_RATE: 15
#EF_PROCESSOR_ENRICH_IPADDR_METADATA_USERDEF_PATH: ""
#EF_PROCESSOR_ENRICH_IPADDR_TTL: 7200
#EF_PROCESSOR_ENRICH_JOIN_ASN: "true"
#EF_PROCESSOR_ENRICH_JOIN_CLOUD: "true"
#EF_PROCESSOR_ENRICH_JOIN_GEOIP: "true"
#EF_PROCESSOR_ENRICH_JOIN_NETATTR: "true"
#EF_PROCESSOR_ENRICH_JOIN_SEC: "true"
#EF_PROCESSOR_ENRICH_JOIN_SUBNETATTR: "true"
#EF_PROCESSOR_ENRICH_NETIF_FLOW_OPTIONS_ENABLE: "true"
#EF_PROCESSOR_ENRICH_NETIF_METADATA_ENABLE: "false"
#EF_PROCESSOR_ENRICH_NETIF_METADATA_REFRESH_RATE: 15
#EF_PROCESSOR_ENRICH_NETIF_METADATA_USERDEF_PATH: ""
#EF_PROCESSOR_ENRICH_NETIF_SNMP_ACCESS_ENABLE: "false"
#EF_PROCESSOR_ENRICH_NETIF_SNMP_ACCESS_PATH: /etc/elastiflow/settings/snmp_access.yml
#EF_PROCESSOR_ENRICH_NETIF_SNMP_ACCESS_REFRESH_RATE: 15
#EF_PROCESSOR_ENRICH_NETIF_SNMP_COMMUNITIES: public
#EF_PROCESSOR_ENRICH_NETIF_SNMP_ENABLE: "false"
#EF_PROCESSOR_ENRICH_NETIF_SNMP_PORT: 161
#EF_PROCESSOR_ENRICH_NETIF_SNMP_RETRIES: 1
#EF_PROCESSOR_ENRICH_NETIF_SNMP_TIMEOUT: 2
#EF_PROCESSOR_ENRICH_NETIF_SNMP_V3_AUTHENTICATION_PASSPHRASE: ""
#EF_PROCESSOR_ENRICH_NETIF_SNMP_V3_AUTHENTICATION_PROTOCOL: noauth
#EF_PROCESSOR_ENRICH_NETIF_SNMP_V3_PRIVACY_PASSPHRASE: ""
#EF_PROCESSOR_ENRICH_NETIF_SNMP_V3_PRIVACY_PROTOCOL: nopriv
#EF_PROCESSOR_ENRICH_NETIF_SNMP_V3_USERNAME: ""
#EF_PROCESSOR_ENRICH_NETIF_SNMP_VERSION: 2
#EF_PROCESSOR_ENRICH_NETIF_TTL: 7200
#EF_PROCESSOR_ENRICH_OPTION_ENUM_TTL: 7200
#EF_PROCESSOR_ENRICH_SAMPLERATE_CACHE_SIZE: 32768
#EF_PROCESSOR_ENRICH_SAMPLERATE_USERDEF_ENABLE: "false"
#EF_PROCESSOR_ENRICH_SAMPLERATE_USERDEF_OVERRIDE: "false"
#EF_PROCESSOR_ENRICH_SAMPLERATE_USERDEF_PATH: /etc/elastiflow/settings/sample_rate.yml
#EF_PROCESSOR_ENRICH_TOTALS_IF_NO_DELTAS: "false"
#EF_PROCESSOR_EXPAND_CLISRV: "true"
#EF_PROCESSOR_EXPAND_CLISRV_NO_L4_PORTS: "true"
#EF_PROCESSOR_IFA_ENABLE: "false"
#EF_PROCESSOR_IFA_POOL_SIZE: 0
#EF_PROCESSOR_IFA_QUEUE_SIZE: 64
#EF_PROCESSOR_KEEP_CPU_TICKS: "false"
#EF_PROCESSOR_PERCENT_NORM: 100
#EF_PROCESSOR_POOL_SIZE: 0
#EF_PROCESSOR_TIMESTAMP_PRECISION: ms
#EF_PROCESSOR_TRANSLATE_KEEP_IDS: default
#EF_OUTPUT_CRIBL_ADDRESSES: 127.0.0.1:10080
#EF_OUTPUT_CRIBL_BATCH_DEADLINE: 2000
#EF_OUTPUT_CRIBL_BATCH_MAX_BYTES: 8388608
#EF_OUTPUT_CRIBL_DROP_FIELDS: ""
#EF_OUTPUT_CRIBL_ENABLE: "false"
#EF_OUTPUT_CRIBL_TLS_CA_CERT_FILEPATH: ""
#EF_OUTPUT_CRIBL_TLS_ENABLE: "true"
#EF_OUTPUT_CRIBL_TLS_SKIP_VERIFICATION: "false"
#EF_OUTPUT_CRIBL_TOKEN: ""
#EF_OUTPUT_GENERIC_HTTP_ADDRESSES: 127.0.0.1:8888
#EF_OUTPUT_GENERIC_HTTP_BATCH_DEADLINE: 2000
#EF_OUTPUT_GENERIC_HTTP_BATCH_MAX_BYTES: 8388608
#EF_OUTPUT_GENERIC_HTTP_DROP_FIELDS: ""
#EF_OUTPUT_GENERIC_HTTP_ECS_ENABLE: "false"
#EF_OUTPUT_GENERIC_HTTP_ENABLE: "false"
#EF_OUTPUT_GENERIC_HTTP_PASSWORD: ""
#EF_OUTPUT_GENERIC_HTTP_TIMESTAMP_SOURCE: collect
#EF_OUTPUT_GENERIC_HTTP_TLS_CA_CERT_FILEPATH: ""
#EF_OUTPUT_GENERIC_HTTP_TLS_ENABLE: "false"
#EF_OUTPUT_GENERIC_HTTP_TLS_SKIP_VERIFICATION: "false"
#EF_OUTPUT_GENERIC_HTTP_USERNAME: ""
#EF_OUTPUT_ELASTICSEARCH_ADDRESSES: 127.0.0.1:9200
#EF_OUTPUT_ELASTICSEARCH_ALLOWED_RECORD_TYPES: as_path_hop,flow_option,flow,ifa_hop,telemetry
#EF_OUTPUT_ELASTICSEARCH_API_KEY: ""
#EF_OUTPUT_ELASTICSEARCH_BATCH_DEADLINE: 2000
#EF_OUTPUT_ELASTICSEARCH_BATCH_MAX_BYTES: 8388608
#EF_OUTPUT_ELASTICSEARCH_CLIENT_CA_CERT_FILEPATH: ""
#EF_OUTPUT_ELASTICSEARCH_CLIENT_CERT_FILEPATH: ""
#EF_OUTPUT_ELASTICSEARCH_CLIENT_KEY_FILEPATH: ""
#EF_OUTPUT_ELASTICSEARCH_CLOUD_ID: ""
#EF_OUTPUT_ELASTICSEARCH_DROP_FIELDS: ""
#EF_OUTPUT_ELASTICSEARCH_ECS_ENABLE: "false"
#EF_OUTPUT_ELASTICSEARCH_ENABLE: "false"
#EF_OUTPUT_ELASTICSEARCH_INDEX_PERIOD: rollover
#EF_OUTPUT_ELASTICSEARCH_INDEX_SUFFIX: ""
#EF_OUTPUT_ELASTICSEARCH_INDEX_TEMPLATE_CODEC: best_compression
#EF_OUTPUT_ELASTICSEARCH_INDEX_TEMPLATE_ENABLE: "true"
#EF_OUTPUT_ELASTICSEARCH_INDEX_TEMPLATE_ILM_LIFECYCLE: elastiflow
#EF_OUTPUT_ELASTICSEARCH_INDEX_TEMPLATE_OVERWRITE: "true"
#EF_OUTPUT_ELASTICSEARCH_INDEX_TEMPLATE_PIPELINE_DEFAULT: _none
#EF_OUTPUT_ELASTICSEARCH_INDEX_TEMPLATE_PIPELINE_FINAL: _none
#EF_OUTPUT_ELASTICSEARCH_INDEX_TEMPLATE_REFRESH_INTERVAL: 10s
#EF_OUTPUT_ELASTICSEARCH_INDEX_TEMPLATE_REPLICAS: 1
#EF_OUTPUT_ELASTICSEARCH_INDEX_TEMPLATE_SHARDS: 3
#EF_OUTPUT_ELASTICSEARCH_MAX_RETRIES: 3
#EF_OUTPUT_ELASTICSEARCH_PASSWORD: changeme
#EF_OUTPUT_ELASTICSEARCH_RETRY_BACKOFF: 1000
#EF_OUTPUT_ELASTICSEARCH_RETRY_ENABLE: "true"
#EF_OUTPUT_ELASTICSEARCH_RETRY_ON_TIMEOUT_ENABLE: "true"
#EF_OUTPUT_ELASTICSEARCH_TIMESTAMP_SOURCE: collect
#EF_OUTPUT_ELASTICSEARCH_TLS_CA_CERT_FILEPATH: ""
#EF_OUTPUT_ELASTICSEARCH_TLS_ENABLE: "false"
#EF_OUTPUT_ELASTICSEARCH_TLS_SKIP_VERIFICATION: "false"
#EF_OUTPUT_ELASTICSEARCH_TSDS_ENABLE: "false"
#EF_OUTPUT_ELASTICSEARCH_USERNAME: elastic
EF_OUTPUT_OPENSEARCH_ADDRESSES: 10.0.40.90:9200
#EF_OUTPUT_OPENSEARCH_ALLOWED_RECORD_TYPES: as_path_hop,flow_option,flow,ifa_hop,telemetry
#EF_OUTPUT_OPENSEARCH_AWS_ACCESS_KEY: ""
#EF_OUTPUT_OPENSEARCH_AWS_REGION: ""
#EF_OUTPUT_OPENSEARCH_AWS_SECRET_KEY: ""
#EF_OUTPUT_OPENSEARCH_BATCH_DEADLINE: 2000
#EF_OUTPUT_OPENSEARCH_BATCH_MAX_BYTES: 8388608
#EF_OUTPUT_OPENSEARCH_CLIENT_CA_CERT_FILEPATH: ""
#EF_OUTPUT_OPENSEARCH_CLIENT_CERT_FILEPATH: ""
#EF_OUTPUT_OPENSEARCH_CLIENT_KEY_FILEPATH: ""
#EF_OUTPUT_OPENSEARCH_DROP_FIELDS: ""
EF_OUTPUT_OPENSEARCH_ECS_ENABLE: "true"
EF_OUTPUT_OPENSEARCH_ENABLE: "true"
#EF_OUTPUT_OPENSEARCH_INDEX_PERIOD: daily
#EF_OUTPUT_OPENSEARCH_INDEX_SUFFIX: ""
#EF_OUTPUT_OPENSEARCH_INDEX_TEMPLATE_CODEC: best_compression
#EF_OUTPUT_OPENSEARCH_INDEX_TEMPLATE_ENABLE: "true"
#EF_OUTPUT_OPENSEARCH_INDEX_TEMPLATE_ISM_POLICY: elastiflow
#EF_OUTPUT_OPENSEARCH_INDEX_TEMPLATE_OVERWRITE: "true"
#EF_OUTPUT_OPENSEARCH_INDEX_TEMPLATE_PIPELINE_DEFAULT: _none
#EF_OUTPUT_OPENSEARCH_INDEX_TEMPLATE_PIPELINE_FINAL: _none
#EF_OUTPUT_OPENSEARCH_INDEX_TEMPLATE_REFRESH_INTERVAL: 10s
#EF_OUTPUT_OPENSEARCH_INDEX_TEMPLATE_REPLICAS: 1
#EF_OUTPUT_OPENSEARCH_INDEX_TEMPLATE_SHARDS: 3
#EF_OUTPUT_OPENSEARCH_MAX_RETRIES: 3
EF_OUTPUT_OPENSEARCH_PASSWORD: "REMOVED"
#EF_OUTPUT_OPENSEARCH_RETRY_BACKOFF: 1000
#EF_OUTPUT_OPENSEARCH_RETRY_ENABLE: "true"
#EF_OUTPUT_OPENSEARCH_RETRY_ON_TIMEOUT_ENABLE: "true"
#EF_OUTPUT_OPENSEARCH_TIMESTAMP_SOURCE: collect
EF_OUTPUT_OPENSEARCH_TLS_CA_CERT_FILEPATH: ""
EF_OUTPUT_OPENSEARCH_TLS_ENABLE: "true"
EF_OUTPUT_OPENSEARCH_TLS_SKIP_VERIFICATION: "true"
EF_OUTPUT_OPENSEARCH_USERNAME: "admin"
#EF_OUTPUT_SPLUNK_HEC_ADDRESSES: 127.0.0.1:8088
#EF_OUTPUT_SPLUNK_HEC_BATCH_DEADLINE: 2000
#EF_OUTPUT_SPLUNK_HEC_BATCH_MAX_BYTES: 8388608
#EF_OUTPUT_SPLUNK_HEC_CIM_ENABLE: "false"
#EF_OUTPUT_SPLUNK_HEC_DROP_FIELDS: ""
#EF_OUTPUT_SPLUNK_HEC_ENABLE: "false"
#EF_OUTPUT_SPLUNK_HEC_TLS_CA_CERT_FILEPATH: ""
#EF_OUTPUT_SPLUNK_HEC_TLS_ENABLE: "true"
#EF_OUTPUT_SPLUNK_HEC_TLS_SKIP_VERIFICATION: "false"
#EF_OUTPUT_SPLUNK_HEC_TOKEN: ""
#EF_OUTPUT_KAFKA_ALLOWED_RECORD_TYPES: as_path_hop,flow_option,flow,ifa_hop,telemetry
#EF_OUTPUT_KAFKA_BROKERS: 127.0.0.1:9092
#EF_OUTPUT_KAFKA_CLIENT_ID: elastiflow-flowcoll
#EF_OUTPUT_KAFKA_DROP_FIELDS: ""
#EF_OUTPUT_KAFKA_ECS_ENABLE: "false"
#EF_OUTPUT_KAFKA_ENABLE: "false"
#EF_OUTPUT_KAFKA_FLAT_RECORD_ENABLE: "true"
#EF_OUTPUT_KAFKA_PARTITION_KEY: flow.export.ip.addr
#EF_OUTPUT_KAFKA_PRODUCER_COMPRESSION: 3
#EF_OUTPUT_KAFKA_PRODUCER_COMPRESSION_LEVEL: -1000
#EF_OUTPUT_KAFKA_PRODUCER_FLUSH_BYTES: 1000000
#EF_OUTPUT_KAFKA_PRODUCER_FLUSH_FREQUENCY: 1000
#EF_OUTPUT_KAFKA_PRODUCER_FLUSH_MAX_MESSAGES: 0
#EF_OUTPUT_KAFKA_PRODUCER_FLUSH_MESSAGES: 1024
#EF_OUTPUT_KAFKA_PRODUCER_MAX_MESSAGE_BYTES: 1000000
#EF_OUTPUT_KAFKA_PRODUCER_REQUIRED_ACKS: 1
#EF_OUTPUT_KAFKA_PRODUCER_RETRY_BACKOFF: 100
#EF_OUTPUT_KAFKA_PRODUCER_RETRY_MAX: 3
#EF_OUTPUT_KAFKA_PRODUCER_TIMEOUT: 10
#EF_OUTPUT_KAFKA_RACK_ID: ""
#EF_OUTPUT_KAFKA_RECORD_TYPE_TOPICS_ENABLE: "false"
#EF_OUTPUT_KAFKA_SASL_ENABLE: "false"
#EF_OUTPUT_KAFKA_SASL_PASSWORD: ""
#EF_OUTPUT_KAFKA_SASL_USERNAME: ""
#EF_OUTPUT_KAFKA_TIMEOUT: 30
#EF_OUTPUT_KAFKA_TIMESTAMP_SOURCE: collect
#EF_OUTPUT_KAFKA_TLS_CA_CERT_FILEPATH: ""
#EF_OUTPUT_KAFKA_TLS_CERT_FILEPATH: ""
#EF_OUTPUT_KAFKA_TLS_ENABLE: "false"
#EF_OUTPUT_KAFKA_TLS_KEY_FILEPATH: ""
#EF_OUTPUT_KAFKA_TLS_SKIP_VERIFICATION: "false"
#EF_OUTPUT_KAFKA_TOPIC: elastiflow-flow-codex
#EF_OUTPUT_KAFKA_TOPIC_VERSION: 1.0
#EF_OUTPUT_KAFKA_VERSION: 1.0.0
#EF_OUTPUT_MONITOR_ENABLE: "false"
#EF_OUTPUT_MONITOR_INTERVAL: 300
#EF_OUTPUT_STDOUT_ENABLE: "false"
#EF_OUTPUT_STDOUT_FORMAT: json_pretty
