Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 35 additions & 10 deletions synapse/models/dns.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,33 +272,58 @@ async def _normPyStr(self, valu, view=None):
)),

('inet:dns:query', {}, (
('client', ('inet:client', {}), {'computed': True, }),
('name', ('inet:dns:name', {}), {'computed': True, }),
('client', ('inet:client', {}), {
'computed': True,
'doc': 'The client that performed the DNS query.'}),

('name', ('inet:dns:name', {}), {
'computed': True,
'doc': 'The DNS query name string.'}),

('name:ip', ('inet:ip', {}), {
'computed': True,
'doc': 'The IP address in the DNS query name string.',
'prevnames': ('name:ipv4', 'name:ipv6')}),

('name:fqdn', ('inet:fqdn', {}), {}),
('type', ('int', {}), {'computed': True, }),
('name:fqdn', ('inet:fqdn', {}), {
'computed': True,
'doc': 'The FQDN in the DNS query name string.'}),

('type', ('int', {}), {
'computed': True,
'doc': 'The type of record that was queried.'}),
)),

('inet:dns:request', {}, (

('query', ('inet:dns:query', {}), {}),
('query:name', ('inet:dns:name', {}), {}),
('query', ('inet:dns:query', {}), {
'doc': 'The DNS query contained in the request.'}),

('query:name', ('inet:dns:name', {}), {
'doc': 'The DNS query name string in the request.'}),

('query:name:ip', ('inet:ip', {}), {
'doc': 'The IP address in the DNS query name string.',
'prevnames': ('query:name:ipv4', 'query:name:ipv6')}),

('query:name:fqdn', ('inet:fqdn', {}), {}),
('query:type', ('int', {}), {}),
('query:name:fqdn', ('inet:fqdn', {}), {
'doc': 'The FQDN in the DNS query name string.'}),

('query:type', ('int', {}), {
'doc': 'The type of record requested in the query.'}),

('reply:code', ('int', {'enums': dnsreplycodes, 'enums:strict': False}), {
'doc': 'The DNS server response code.'}),
)),

('inet:dns:answer', {}, (

('ttl', ('int', {}), {}),
('request', ('inet:dns:request', {}), {}),
('ttl', ('int', {}), {
'doc': 'The time to live value of the DNS record in the response.'}),

('request', ('inet:dns:request', {}), {
'doc': 'The DNS request that was answered.'}),

('record', ('inet:dns:record', {}), {
'doc': 'The DNS record returned by the lookup.',
'prevnames': ('a', 'aaaa', 'cname', 'mx', 'ns', 'rev', 'soa', 'txt')}),
Expand Down
3 changes: 2 additions & 1 deletion synapse/models/geopol.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,8 @@
'prevnames': ('isonum',),
'doc': 'The ISO 3166 Numeric-3 country code.'}),

('tld', ('inet:fqdn', {}), {}),
('tld', ('inet:fqdn', {}), {
'doc': 'The top-level domain for the country.'}),

('name', ('meta:name', {}), {
'alts': ('names',),
Expand Down
43 changes: 32 additions & 11 deletions synapse/models/gov.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,17 +78,38 @@

'forms': (
('gov:us:cage', {}, (
('org', ('ou:org', {}), {'doc': 'The organization which was issued the CAGE code.'}),
('name0', ('meta:name', {}), {'doc': 'The name of the organization.'}),
('name1', ('str', {'lower': True}), {'doc': 'Name Part 1.'}),
('street', ('str', {'lower': True}), {}),
('city', ('str', {'lower': True}), {}),
('state', ('str', {'lower': True}), {}),
('zip', ('gov:us:zip', {}), {}),
('cc', ('iso:3166:alpha2', {}), {}),
('country', ('str', {'lower': True}), {}),
('phone0', ('tel:phone', {}), {}),
('phone1', ('tel:phone', {}), {}),
('org', ('ou:org', {}), {
'doc': 'The organization which was issued the CAGE code.'}),

('name0', ('meta:name', {}), {
'doc': 'The name of the organization.'}),

('name1', ('str', {'lower': True}), {
'doc': 'Name Part 1.'}),

('street', ('str', {'lower': True}), {
'doc': 'The street in the CAGE code record.'}),

('city', ('str', {'lower': True}), {
'doc': 'The city in the CAGE code record.'}),

('state', ('str', {'lower': True}), {
'doc': 'The state in the CAGE code record.'}),

('zip', ('gov:us:zip', {}), {
'doc': 'The zip code in the CAGE code record.'}),

('cc', ('iso:3166:alpha2', {}), {
'doc': 'The country code in the CAGE code record.'}),

('country', ('str', {'lower': True}), {
'doc': 'The country in the CAGE code record.'}),

('phone0', ('tel:phone', {}), {
'doc': 'The primary phone number in the CAGE code record.'}),

('phone1', ('tel:phone', {}), {
'doc': 'The alternate phone number in the CAGE code record.'}),
)),

('gov:us:ssn', {}, []),
Expand Down
18 changes: 13 additions & 5 deletions synapse/models/inet.py
Original file line number Diff line number Diff line change
Expand Up @@ -2348,7 +2348,6 @@ async def _onSetWhoisText(node):

('inet:http:request', {}, (


('method', ('str', {}), {
'doc': 'The HTTP request method string.'}),

Expand All @@ -2373,12 +2372,21 @@ async def _onSetWhoisText(node):
('cookies', ('array', {'type': 'inet:http:cookie'}), {
'doc': 'An array of HTTP cookie values parsed from the "Cookies:" header in the request.'}),

('response:time', ('time', {}), {}),
('response:code', ('int', {}), {}),
('response:reason', ('str', {}), {}),
('response:time', ('time', {}), {
'doc': 'The time a response to the request was received.'}),

('response:code', ('int', {}), {
'doc': 'The HTTP response code received.'}),

('response:reason', ('str', {}), {
'doc': 'The HTTP response reason phrase received.'}),

('response:headers', ('array', {'type': 'inet:http:response:header', 'uniq': False, 'sorted': False}), {
'doc': 'An array of HTTP headers from the response.'}),
('response:body', ('file:bytes', {}), {}),

('response:body', ('file:bytes', {}), {
'doc': 'The HTTP response body received.'}),

('session', ('inet:http:session', {}), {
'doc': 'The HTTP session this request was part of.'}),
)),
Expand Down
7 changes: 5 additions & 2 deletions synapse/models/orgs.py
Original file line number Diff line number Diff line change
Expand Up @@ -599,8 +599,11 @@
'doc': 'A base tag used to encode assessments made by the organization.'}),
)),
('ou:team', {}, (
('org', ('ou:org', {}), {}),
('name', ('meta:name', {}), {}),
('org', ('ou:org', {}), {
'doc': 'The organization that the team is associated with.'}),

('name', ('meta:name', {}), {
'doc': 'The name of the team.'}),
)),

('ou:asset:type:taxonomy', {}, ()),
Expand Down
37 changes: 27 additions & 10 deletions synapse/models/telco.py
Original file line number Diff line number Diff line change
Expand Up @@ -355,7 +355,8 @@ async def _normPyInt(self, valu, view=None):

('tel:mob:telem', {}, (

('time', ('time', {}), {}),
('time', ('time', {}), {
'doc': 'The time that the telemetry sample was taken.'}),

('http:request', ('inet:http:request', {}), {
'doc': 'The HTTP request that the telemetry was extracted from.'}),
Expand All @@ -364,17 +365,28 @@ async def _normPyInt(self, valu, view=None):
'doc': 'The host that generated the mobile telemetry data.'}),

# telco specific data
('cell', ('tel:mob:cell', {}), {}),
('imsi', ('tel:mob:imsi', {}), {}),
('imei', ('tel:mob:imei', {}), {}),
('phone', ('tel:phone', {}), {}),
('cell', ('tel:mob:cell', {}), {
'doc': 'The mobile cell site where the telemetry sample was taken.'}),

('imsi', ('tel:mob:imsi', {}), {
'doc': 'The IMSI of the device associated with the mobile telemetry sample.'}),

('imei', ('tel:mob:imei', {}), {
'doc': 'The IMEI of the device associated with the mobile telemetry sample.'}),

('phone', ('tel:phone', {}), {
'doc': 'The phone number of the device associated with the mobile telemetry sample.'}),

# inet protocol addresses
('mac', ('inet:mac', {}), {}),
('mac', ('inet:mac', {}), {
'doc': 'The MAC address of the device associated with the mobile telemetry sample.'}),

('ip', ('inet:ip', {}), {
'doc': 'The IP address of the device associated with the mobile telemetry sample.',
'prevnames': ('ipv4', 'ipv6')}),

('wifi:ap', ('inet:wifi:ap', {}), {
'doc': 'The Wi-Fi AP associated with the mobile telemetry sample.',
'prevnames': ('wifi',)}),

# host specific data
Expand All @@ -383,16 +395,21 @@ async def _normPyInt(self, valu, view=None):

# FIXME contact prop or interface?
# User related data
('name', ('meta:name', {}), {}),
('email', ('inet:email', {}), {}),
('name', ('meta:name', {}), {
'doc': 'The user name associated with the mobile telemetry sample.'}),

('email', ('inet:email', {}), {
'doc': 'The email address associated with the mobile telemetry sample.'}),

('account', ('inet:service:account', {}), {
'doc': 'The service account which is associated with the tracked device.'}),

# reporting related data
('app', ('it:software', {}), {}),
('app', ('it:software', {}), {
'doc': 'The app used to report the mobile telemetry sample.'}),

('data', ('data', {}), {}),
('data', ('data', {}), {
'doc': 'Data from the mobile telemetry sample.'}),
# any other fields may be refs...
)),
)
Expand Down
30 changes: 0 additions & 30 deletions synapse/tests/test_model_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,52 +151,22 @@ async def test_model_base_rules(self):

async def test_model_doc_strings(self):

self.skip('FIXME - do we wanna just mop these up?')
async with self.getTestCore() as core:

nodes = await core.nodes('syn:type:doc="" -:ctor^="synapse.tests"')
self.len(0, nodes)

SYN_6315 = [
'inet:dns:query:client', 'inet:dns:query:name', 'inet:dns:query:name:ip',
'inet:dns:query:name:fqdn', 'inet:dns:query:type',
'inet:dns:request:time', 'inet:dns:request:query', 'inet:dns:request:query:name',
'inet:dns:request:query:name:ip',
'inet:dns:request:query:name:fqdn', 'inet:dns:request:query:type',
'inet:dns:request:server', 'inet:dns:answer:ttl', 'inet:dns:answer:request',
'ou:team:org', 'ou:team:name',
'entity:contact:asof', 'pol:country:iso2', 'pol:country:iso3', 'pol:country:isonum',
'pol:country:tld', 'tel:mob:carrier:mcc', 'tel:mob:carrier:mnc',
'tel:mob:telem:time', 'tel:mob:telem:latlong', 'tel:mob:telem:cell',
'tel:mob:telem:cell:carrier', 'tel:mob:telem:imsi', 'tel:mob:telem:imei',
'tel:mob:telem:phone', 'tel:mob:telem:mac', 'tel:mob:telem:ip',
'tel:mob:telem:wifi:ap', 'tel:mob:telem:wifi:ap:ssid', 'tel:mob:telem:wifi:ap:bssid',
'tel:mob:telem:name', 'tel:mob:telem:email',
'tel:mob:telem:app', 'tel:mob:telem:data',
'inet:http:request:response:time', 'inet:http:request:response:code',
'inet:http:request:response:reason', 'inet:http:request:response:body',
'gov:us:cage:street', 'gov:us:cage:city', 'gov:us:cage:state', 'gov:us:cage:zip',
'gov:us:cage:cc', 'gov:us:cage:country', 'gov:us:cage:phone0', 'gov:us:cage:phone1',
'biz:rfp:requirements',
]

nodes = await core.nodes('syn:prop:doc=""')
keep = []
skip = []
for node in nodes:
name = node.ndef[1]

if name in SYN_6315:
skip.append(node.form.name)
continue

if name.startswith('test:'):
continue

keep.append(node)

self.len(0, keep, msg=[node.ndef[1] for node in keep])
self.sorteq(SYN_6315, skip)

for edge in core.model.edges.values():
doc = edge.edgeinfo.get('doc')
Expand Down