Commit a175df87 authored by Peter V. Saveliev's avatar Peter V. Saveliev

docs: ndb intro, schema, sources

parent e7ee7e93
.. ipdb:
.. _ipdb:
.. automodule:: pyroute2.ipdb.main
:members:
......
.. iproute:
.. _iproute:
IPRoute module
==============
......
.. ndb:
NDB module
==========
.. automodule:: pyroute2.ndb.main
Reference
---------
.. toctree::
:maxdepth: 2
ndb_schema
ndb_sources
.. ndb_schema:
.. automodule:: pyroute2.ndb.dbschema
.. ndb_sources:
.. automodule:: pyroute2.ndb.source
.. netns:
.. _netns:
NetNS module
============
.. automodule:: pyroute2.netns
:members:
......
.. remote:
.. _remote:
RemoteIPRoute
-------------
......
'''
Database
========
Backends
--------
NDB stores all the records in an SQL database. By default it uses
the SQLite3 module, which is a part of the Python stdlib, so no
extra packages are required::
# SQLite3 -- simple in-memory DB
ndb = NDB()
# SQLite3 -- file DB
ndb = NDB(db_provider='sqlite3', db_spec='test.db')
It is also possible to use a PostgreSQL database via psycopg2
module::
# PostgreSQL -- local DB
ndb = NDB(db_provider='psycopg2',
db_spec={'dbname': 'test'})
# PostgreSQL -- remote DB
ndb = NDB(db_provider='psycopg2',
db_spec={'dbname': 'test',
'host': 'db1.example.com'})
SQL schema
----------
A file based SQLite3 DB or PostgreSQL may be useful for inspection
of the collected data. Here is an example schema::
rtnl=# \\dt
List of relations
Schema | Name | Type | Owner
--------+-----------------+-------+-------
public | addresses | table | root
public | ifinfo_bond | table | root
public | ifinfo_bridge | table | root
public | ifinfo_gre | table | root
public | ifinfo_vlan | table | root
public | ifinfo_vrf | table | root
public | ifinfo_vti | table | root
public | ifinfo_vti6 | table | root
public | ifinfo_vxlan | table | root
public | interfaces | table | root
public | neighbours | table | root
public | nh | table | root
public | routes | table | root
public | sources | table | root
public | sources_options | table | root
(15 rows)
rtnl=# select f_index, f_ifla_ifname from interfaces;
f_index | f_ifla_ifname
---------+---------------
1 | lo
2 | eth0
28 | ip_vti0
31 | ip6tnl0
32 | ip6_vti0
36445 | br0
11434 | dummy0
3 | eth1
(8 rows)
rtnl=# select f_index, f_ifla_br_stp_state from ifinfo_bridge;
f_index | f_ifla_br_stp_state
---------+---------------------
36445 | 0
(1 row)
There are also some useful views, that join `ifinfo` tables with
`interfaces`::
rtnl=# \\dv
List of relations
Schema | Name | Type | Owner
--------+--------+------+-------
public | bond | view | root
public | bridge | view | root
public | gre | view | root
public | vlan | view | root
public | vrf | view | root
public | vti | view | root
public | vti6 | view | root
public | vxlan | view | root
(8 rows)
'''
import time
import uuid
import struct
......@@ -358,7 +452,7 @@ class DBSchema(object):
# service tables
#
self.execute('''
DROP TABLE IF EXISTS options
DROP TABLE IF EXISTS sources_options
''')
self.execute('''
DROP TABLE IF EXISTS sources
......@@ -369,7 +463,7 @@ class DBSchema(object):
f_kind TEXT NOT NULL)
''')
self.execute('''
CREATE TABLE IF NOT EXISTS options
CREATE TABLE IF NOT EXISTS sources_options
(f_target TEXT NOT NULL,
f_name TEXT NOT NULL,
f_type TEXT NOT NULL,
......
'''
NDB module
==========
A high-level RTNL management module. It provides a database
of actual network settings, the database is being updated in
the real time.
NDB is a high level network management module. IT allows to manage interfaces,
routes, addresses etc. of connected systems, containers and network
namespaces.
NDB work with remote systems via ssh, in that case
`mitogen <https://github.com/dw/mitogen>`_ module is required. It is
possible to connect also OpenBSD and FreeBSD systems, but in read-only
mode for now.
Quick start
-----------
......@@ -57,31 +60,6 @@ Key NDB features:
* Fault tolerance and memory consumtion limits
* Transactions
DB backends
-----------
NDB supports different DB providers, for now SQLite3 and PostgreSQL.
The default backend is SQLite3 which is a part of the Python standard
library, no extra dependencies are employed in that case::
# SQLite3 -- simple in-memory DB
ndb = NDB()
# SQLite3 -- file DB
ndb = NDB(db_provider='sqlite3', db_spec='test.db')
The PostgreSQL backend requires psycopg2 module::
# PostgreSQL -- local DB
ndb = NDB(db_provider='psycopg2',
db_spec={'dbname': 'test'})
# PostgreSQL -- remote DB
ndb = NDB(db_provider='psycopg2',
db_spec={'dbname': 'test',
'host': 'db1.example.com'})
'''
import gc
import json
......@@ -732,7 +710,7 @@ class NDB(object):
def execute(self, *argv, **kwarg):
return self.schema.execute(*argv, **kwarg)
def close(self):
def close(self, flush=False):
with self._global_lock:
if self._dbm_shutdown.is_set():
return
......@@ -752,7 +730,7 @@ class NDB(object):
self._event_queue.put(('localhost', (ShutdownException(), )))
# release all the sources
for target, source in self.sources.cache.items():
source.close()
source.close(flush)
# close the database
self.schema.commit()
self.schema.close()
......
'''
RTNL sources
============
Local RTNL
----------
Local RTNL source is a simple `IPRoute` instance. By default NDB
starts with one local RTNL source names `localhost`::
>>> ndb = NDB()
>>> ndb.sources.details()
{'kind': u'local', u'nlm_generator': 1, 'target': u'localhost'}
>>> ndb.sources['localhost']
[running] <IPRoute {'nlm_generator': 1}>
The `localhost` RTNL source starts an additional async cache thread.
The `nlm_generator` option means that instead of collections the
`IPRoute` object returns generators, so `IPRoute` responses will not
consume memory regardless of the RTNL objects number::
>>> ndb.sources['localhost'].nl.link('dump')
<generator object _match at 0x7fa444961e10>
See also: :ref:`iproute`
Network namespaces
------------------
There are two ways to connect additional sources to an NDB instance.
One is to specify sources when creating an NDB object::
ndb = NDB(sources=[{'target': 'localhost'}, {'netns': 'test01'}])
Another way is to call `ndb.sources.add()` method::
ndb.sources.add(netns='test01')
This syntax: `{target': 'localhost'}` and `{'netns': 'test01'}` is the
short form. The full form would be::
{'target': 'localhost', # the label for the DB
'kind': 'local', # use IPRoute class to start the source
'nlm_generator': 1} #
{'target': 'test01', # the label
'kind': 'netns', # use NetNS class
'netns': 'test01'} #
See also: :ref:`netns`
Remote systems
--------------
It is possible also to connect to remote systems using SSH. In order to
use this kind of sources it is required to install the
`mitogen <https://github.com/dw/mitogen>`_ module. The `remote` kind
of sources uses the `RemoteIPRoute` class. The short form::
ndb.sources.add(hostname='worker1.example.com')
In some more extended form::
ndb.sources.add(**{'target': 'worker1.example.com',
'kind': 'remote',
'hostname': 'worker1.example.com',
'username': 'jenkins',
'check_host_keys': False})
See also: :ref:`remote`
'''
import time
import threading
from pyroute2 import IPRoute
......@@ -64,10 +136,8 @@ class Source(dict):
for key, value in spec.items():
vtype = 'int' if isinstance(value, int) else 'str'
self.ndb.schema.execute('''
INSERT INTO options (f_target,
f_name,
f_type,
f_value)
INSERT INTO sources_options
(f_target, f_name, f_type, f_value)
VALUES (%s, %s, %s, %s)
''' % (self.ndb.schema.plch,
self.ndb.schema.plch,
......@@ -106,7 +176,7 @@ class Source(dict):
.ndb
.schema
.execute('''
DELETE FROM options WHERE f_target=%s
DELETE FROM sources_options WHERE f_target=%s
''' % self.ndb.schema.plch, (self.target, )))
return self
......@@ -242,7 +312,7 @@ class Source(dict):
self.th.start()
return self
def close(self):
def close(self, flush=True):
with self.lock:
self.log.debug('stopping the source')
self.shutdown.set()
......@@ -254,7 +324,8 @@ class Source(dict):
if self.th is not None:
self.th.join()
self.log.debug('flushing the DB for the target')
self.ndb.schema.flush(self.target)
if flush:
self.ndb.schema.flush(self.target)
def restart(self):
with self.lock:
......@@ -282,7 +353,7 @@ class Source(dict):
(self.target, ))
self['target'], self['kind'] = spec
for spec in self.ndb.schema.fetch('''
SELECT * FROM options
SELECT * FROM sources_options
WHERE f_target = %s
''' % self.ndb.schema.plch,
(self.target, )):
......
'''
NetNS
=====
NetNS objects
=============
A NetNS object is IPRoute-like. It runs in the main network
namespace, but also creates a proxy process running in
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment