Commit 3d87838c authored by Jonathon Duerig's avatar Jonathon Duerig

Added ewma for throughput. Various fixes. Various additional logging....

Added ewma for throughput. Various fixes. Various additional logging. Simplified saturation calculation and moved it to StateSensor
parent 9ba850a2
...@@ -32,13 +32,16 @@ public: ...@@ -32,13 +32,16 @@ public:
if (message.size() <= 0xffff && message.size() > 0) if (message.size() <= 0xffff && message.size() > 0)
{ {
Header prefix; Header prefix;
std::string pathString;
if (dir == FORWARD_PATH) if (dir == FORWARD_PATH)
{ {
prefix.type = EVENT_FORWARD_PATH; prefix.type = EVENT_FORWARD_PATH;
pathString = "FORWARD";
} }
else else
{ {
prefix.type = EVENT_BACKWARD_PATH; prefix.type = EVENT_BACKWARD_PATH;
pathString = "BACKWARD";
} }
prefix.size = message.size(); prefix.size = message.size();
prefix.key = key; prefix.key = key;
...@@ -50,6 +53,8 @@ public: ...@@ -50,6 +53,8 @@ public:
writeMessage(headerBuffer, Header::headerSize); writeMessage(headerBuffer, Header::headerSize);
writeMessage(message.c_str(), message.size()); writeMessage(message.c_str(), message.size());
endMessage(); endMessage();
logWrite(COMMAND_OUTPUT, "(%s,%s): %s",
key.toString().c_str(), pathString.c_str(), message.c_str());
} }
} }
else else
......
...@@ -31,10 +31,6 @@ void DelaySensor::localAck(PacketInfo * packet) ...@@ -31,10 +31,6 @@ void DelaySensor::localAck(PacketInfo * packet)
{ {
Time diff = packet->packetTime - packetHistory->getAckedSendTime(); Time diff = packet->packetTime - packetHistory->getAckedSendTime();
lastDelay = diff.toMilliseconds(); lastDelay = diff.toMilliseconds();
logWrite(SENSOR, "DELAY realDiff: %f seconds, calcDiff: %f", logWrite(SENSOR, "DELAY: %d ms", lastDelay);
packet->packetTime.toDouble()
- packetHistory->getAckedSendTime().toDouble(),
diff.toDouble());
logWrite(SENSOR, "DELAY: %d", lastDelay);
} }
} }
all: magent all: magent
magent: CircularTraffic.o Command.o Connection.o Decayer.o DelaySensor.o DirectInput.o KernelTcp.o MaxDelaySensor.o MinDelaySensor.o PacketSensor.o Sensor.o SensorList.o StateSensor.o ThroughputSensor.o Time.o TrivialCommandOutput.o log.o main.o saveload.o magent: CircularTraffic.o Command.o Connection.o Decayer.o DelaySensor.o DirectInput.o EwmaThroughputSensor.o KernelTcp.o MaxDelaySensor.o MinDelaySensor.o PacketSensor.o Sensor.o SensorList.o StateSensor.o ThroughputSensor.o Time.o TrivialCommandOutput.o log.o main.o saveload.o
g++ -I. -g -Wall CircularTraffic.o Command.o Connection.o Decayer.o DelaySensor.o DirectInput.o KernelTcp.o MaxDelaySensor.o MinDelaySensor.o PacketSensor.o Sensor.o SensorList.o StateSensor.o ThroughputSensor.o Time.o TrivialCommandOutput.o log.o main.o saveload.o -lm -lpcap -o magent g++ -I. -g -Wall CircularTraffic.o Command.o Connection.o Decayer.o DelaySensor.o DirectInput.o EwmaThroughputSensor.o KernelTcp.o MaxDelaySensor.o MinDelaySensor.o PacketSensor.o Sensor.o SensorList.o StateSensor.o ThroughputSensor.o Time.o TrivialCommandOutput.o log.o main.o saveload.o -lm -lpcap -o magent
CircularTraffic.o: CircularTraffic.cc lib.h log.h TrafficModel.h CircularTraffic.h Command.h ConnectionModel.h CircularTraffic.o: CircularTraffic.cc lib.h log.h TrafficModel.h CircularTraffic.h Command.h ConnectionModel.h
g++ -I. -g -Wall -c CircularTraffic.cc g++ -I. -g -Wall -c CircularTraffic.cc
...@@ -21,10 +21,13 @@ DelaySensor.o: DelaySensor.cc lib.h Sensor.h DelaySensor.h PacketSensor.h Time.h ...@@ -21,10 +21,13 @@ DelaySensor.o: DelaySensor.cc lib.h Sensor.h DelaySensor.h PacketSensor.h Time.h
DirectInput.o: DirectInput.cc lib.h log.h CommandInput.h saveload.h DirectInput.h DirectInput.o: DirectInput.cc lib.h log.h CommandInput.h saveload.h DirectInput.h
g++ -I. -g -Wall -c DirectInput.cc g++ -I. -g -Wall -c DirectInput.cc
EwmaThroughputSensor.o: EwmaThroughputSensor.cc lib.h Sensor.h EwmaThroughputSensor.h ThroughputSensor.h CommandOutput.h
g++ -I. -g -Wall -c EwmaThroughputSensor.cc
KernelTcp.o: KernelTcp.cc lib.h log.h KernelTcp.h Command.h KernelTcp.o: KernelTcp.cc lib.h log.h KernelTcp.h Command.h
g++ -I. -g -Wall -c KernelTcp.cc g++ -I. -g -Wall -c KernelTcp.cc
MaxDelaySensor.o: MaxDelaySensor.cc lib.h Sensor.h Decayer.h MaxDelaySensor.h DelaySensor.h log.h saveload.h CommandOutput.h MaxDelaySensor.o: MaxDelaySensor.cc lib.h Sensor.h Decayer.h MaxDelaySensor.h DelaySensor.h log.h saveload.h CommandOutput.h StateSensor.h
g++ -I. -g -Wall -c MaxDelaySensor.cc g++ -I. -g -Wall -c MaxDelaySensor.cc
MinDelaySensor.o: MinDelaySensor.cc lib.h Sensor.h Decayer.h MinDelaySensor.h DelaySensor.h log.h saveload.h CommandOutput.h MinDelaySensor.o: MinDelaySensor.cc lib.h Sensor.h Decayer.h MinDelaySensor.h DelaySensor.h log.h saveload.h CommandOutput.h
...@@ -36,7 +39,7 @@ PacketSensor.o: PacketSensor.cc lib.h Sensor.h PacketSensor.h StateSensor.h ...@@ -36,7 +39,7 @@ PacketSensor.o: PacketSensor.cc lib.h Sensor.h PacketSensor.h StateSensor.h
Sensor.o: Sensor.cc lib.h Sensor.h Sensor.o: Sensor.cc lib.h Sensor.h
g++ -I. -g -Wall -c Sensor.cc g++ -I. -g -Wall -c Sensor.cc
SensorList.o: SensorList.cc lib.h log.h SensorList.h Sensor.h Command.h PacketSensor.h DelaySensor.h MinDelaySensor.h MaxDelaySensor.h ThroughputSensor.h StateSensor.h SensorList.o: SensorList.cc lib.h log.h SensorList.h Sensor.h Command.h PacketSensor.h DelaySensor.h MinDelaySensor.h MaxDelaySensor.h ThroughputSensor.h StateSensor.h EwmaThroughputSensor.h
g++ -I. -g -Wall -c SensorList.cc g++ -I. -g -Wall -c SensorList.cc
StateSensor.o: StateSensor.cc lib.h Sensor.h StateSensor.h StateSensor.o: StateSensor.cc lib.h Sensor.h StateSensor.h
......
...@@ -4,13 +4,15 @@ ...@@ -4,13 +4,15 @@
#include "MaxDelaySensor.h" #include "MaxDelaySensor.h"
#include "DelaySensor.h" #include "DelaySensor.h"
#include "CommandOutput.h" #include "CommandOutput.h"
#include "StateSensor.h"
using namespace std; using namespace std;
MaxDelaySensor::MaxDelaySensor(DelaySensor * newDelay) MaxDelaySensor::MaxDelaySensor(DelaySensor * newDelay, StateSensor * newState)
: maximum(0, 0.01) : maximum(0, 0.01)
, delay(newDelay)
, state(newState)
{ {
delay = newDelay;
} }
void MaxDelaySensor::localSend(PacketInfo *) void MaxDelaySensor::localSend(PacketInfo *)
...@@ -20,7 +22,8 @@ void MaxDelaySensor::localSend(PacketInfo *) ...@@ -20,7 +22,8 @@ void MaxDelaySensor::localSend(PacketInfo *)
void MaxDelaySensor::localAck(PacketInfo * packet) void MaxDelaySensor::localAck(PacketInfo * packet)
{ {
int current = delay->getLastDelay(); int current = delay->getLastDelay();
if (current > maximum && current != 0) logWrite(SENSOR, "current=%d,saturated=%d", current, state->isSaturated());
if (current > maximum && current != 0 && state->isSaturated())
{ {
ostringstream buffer; ostringstream buffer;
buffer << "MAXINQ=" << current; buffer << "MAXINQ=" << current;
......
...@@ -7,17 +7,19 @@ ...@@ -7,17 +7,19 @@
#include "Decayer.h" #include "Decayer.h"
class DelaySensor; class DelaySensor;
class StateSensor;
class MaxDelaySensor : public Sensor class MaxDelaySensor : public Sensor
{ {
public: public:
MaxDelaySensor(DelaySensor * newDelay); MaxDelaySensor(DelaySensor * newDelay, StateSensor * newState);
protected: protected:
virtual void localSend(PacketInfo * packet); virtual void localSend(PacketInfo * packet);
virtual void localAck(PacketInfo * packet); virtual void localAck(PacketInfo * packet);
private: private:
Decayer maximum; Decayer maximum;
DelaySensor * delay; DelaySensor * delay;
StateSensor * state;
}; };
#endif #endif
...@@ -5,16 +5,6 @@ ...@@ -5,16 +5,6 @@
using namespace std; using namespace std;
bool Sensor::isLinkSaturated(PacketInfo * packet)
{
return packet->bufferFull &&
// and *not* in slow start
!(packet->kernel->tcpi_snd_cwnd < packet->kernel->tcpi_snd_ssthresh
|| (static_cast<unsigned int>(htons(packet->tcp->window))
<< packet->kernel->tcpi_rcv_wscale)
<= (packet->kernel->tcpi_unacked * 1448));
}
Sensor::~Sensor() Sensor::~Sensor()
{ {
} }
......
...@@ -11,8 +11,6 @@ ...@@ -11,8 +11,6 @@
class Sensor class Sensor
{ {
public:
bool isLinkSaturated(PacketInfo * packet);
public: public:
virtual ~Sensor(); virtual ~Sensor();
Sensor * getTail(void); Sensor * getTail(void);
......
...@@ -12,6 +12,7 @@ ...@@ -12,6 +12,7 @@
#include "MinDelaySensor.h" #include "MinDelaySensor.h"
#include "MaxDelaySensor.h" #include "MaxDelaySensor.h"
#include "ThroughputSensor.h" #include "ThroughputSensor.h"
#include "EwmaThroughputSensor.h"
using namespace std; using namespace std;
...@@ -72,6 +73,9 @@ void SensorList::addSensor(SensorCommand const & newSensor) ...@@ -72,6 +73,9 @@ void SensorList::addSensor(SensorCommand const & newSensor)
case THROUGHPUT_SENSOR: case THROUGHPUT_SENSOR:
pushThroughputSensor(); pushThroughputSensor();
break; break;
case EWMA_THROUGHPUT_SENSOR:
pushEwmaThroughputSensor();
break;
default: default:
logWrite(ERROR, logWrite(ERROR,
"Incorrect sensor type (%d). Ignoring add sensor command.", "Incorrect sensor type (%d). Ignoring add sensor command.",
...@@ -133,6 +137,7 @@ void SensorList::pushNullSensor(void) ...@@ -133,6 +137,7 @@ void SensorList::pushNullSensor(void)
void SensorList::pushStateSensor(void) void SensorList::pushStateSensor(void)
{ {
// Dependency list // Dependency list
pushNullSensor();
// Dependency check // Dependency check
if (depStateSensor == NULL) if (depStateSensor == NULL)
...@@ -198,9 +203,11 @@ void SensorList::pushMaxDelaySensor(void) ...@@ -198,9 +203,11 @@ void SensorList::pushMaxDelaySensor(void)
{ {
// Dependency list // Dependency list
pushDelaySensor(); pushDelaySensor();
pushStateSensor();
logWrite(SENSOR, "Adding MaxDelaySensor"); logWrite(SENSOR, "Adding MaxDelaySensor");
std::auto_ptr<Sensor> current(new MaxDelaySensor(depDelaySensor)); std::auto_ptr<Sensor> current(new MaxDelaySensor(depDelaySensor,
depStateSensor));
pushSensor(current); pushSensor(current);
} }
...@@ -223,3 +230,13 @@ void SensorList::pushThroughputSensor(void) ...@@ -223,3 +230,13 @@ void SensorList::pushThroughputSensor(void)
depThroughputSensor = newSensor; depThroughputSensor = newSensor;
} }
} }
void SensorList::pushEwmaThroughputSensor(void)
{
// Dependency list
pushThroughputSensor();
logWrite(SENSOR, "Adding EwmaThroughputSensor");
std::auto_ptr<Sensor> current(new EwmaThroughputSensor(depThroughputSensor));
pushSensor(current);
}
...@@ -23,6 +23,7 @@ class DelaySensor; ...@@ -23,6 +23,7 @@ class DelaySensor;
class MinDelaySensor; class MinDelaySensor;
class MaxDelaySensor; class MaxDelaySensor;
class ThroughputSensor; class ThroughputSensor;
class EwmaThroughputSensor;
class SensorList class SensorList
{ {
...@@ -47,6 +48,7 @@ private: ...@@ -47,6 +48,7 @@ private:
void pushMinDelaySensor(void); void pushMinDelaySensor(void);
void pushMaxDelaySensor(void); void pushMaxDelaySensor(void);
void pushThroughputSensor(void); void pushThroughputSensor(void);
void pushEwmaThroughputSensor(void);
private: private:
// Example dependency // Example dependency
NullSensor * depNullSensor; NullSensor * depNullSensor;
......
...@@ -15,11 +15,16 @@ StateSensor::~StateSensor() ...@@ -15,11 +15,16 @@ StateSensor::~StateSensor()
{ {
} }
int StateSensor::getState(void) int StateSensor::getState(void) const
{ {
return state; return state;
} }
bool StateSensor::isSaturated(void) const
{
return saturated;
}
void StateSensor::localSend(PacketInfo * packet) void StateSensor::localSend(PacketInfo * packet)
{ {
if (packet->tcp->syn && state == INITIAL) if (packet->tcp->syn && state == INITIAL)
...@@ -41,6 +46,7 @@ void StateSensor::localSend(PacketInfo * packet) ...@@ -41,6 +46,7 @@ void StateSensor::localSend(PacketInfo * packet)
state = ESTABLISHED; state = ESTABLISHED;
logWrite(SENSOR, "State change to ESTABLISHED"); logWrite(SENSOR, "State change to ESTABLISHED");
} }
calculateSaturated(packet);
} }
void StateSensor::localAck(PacketInfo * packet) void StateSensor::localAck(PacketInfo * packet)
...@@ -59,4 +65,22 @@ void StateSensor::localAck(PacketInfo * packet) ...@@ -59,4 +65,22 @@ void StateSensor::localAck(PacketInfo * packet)
state = ESTABLISHED; state = ESTABLISHED;
logWrite(SENSOR, "State change to ESTABLISHED"); logWrite(SENSOR, "State change to ESTABLISHED");
} }
calculateSaturated(packet);
}
void StateSensor::calculateSaturated(PacketInfo * packet)
{
unsigned int snd_cwnd = packet->kernel->tcpi_snd_cwnd;
unsigned int snd_ssthresh = packet->kernel->tcpi_snd_ssthresh;
unsigned int window = (static_cast<unsigned int>(htons(packet->tcp->window))
<< packet->kernel->tcpi_rcv_wscale);
unsigned int unacked = packet->kernel->tcpi_unacked * 1448;
logWrite(SENSOR, "stateEstablished=%d,bufferFull=%d", state == ESTABLISHED,
packet->bufferFull);
logWrite(SENSOR, "snd_cwnd=%u,snd_ssthresh=%u,window=%u,unacked=%u",
snd_cwnd, snd_ssthresh, window, unacked);
saturated = (state == ESTABLISHED
&& packet->bufferFull
// and *not* in slow start
&& !(snd_cwnd < snd_ssthresh || window <= unacked));
} }
...@@ -22,12 +22,16 @@ public: ...@@ -22,12 +22,16 @@ public:
public: public:
StateSensor(); StateSensor();
virtual ~StateSensor(); virtual ~StateSensor();
int getState(void); int getState(void) const;
bool isSaturated(void) const;
protected: protected:
virtual void localSend(PacketInfo * packet); virtual void localSend(PacketInfo * packet);
virtual void localAck(PacketInfo * packet); virtual void localAck(PacketInfo * packet);
private:
void calculateSaturated(PacketInfo * packet);
private: private:
int state; int state;
bool saturated;
}; };
#endif #endif
...@@ -10,10 +10,10 @@ using namespace std; ...@@ -10,10 +10,10 @@ using namespace std;
ThroughputSensor::ThroughputSensor(PacketSensor * newPacketHistory, ThroughputSensor::ThroughputSensor(PacketSensor * newPacketHistory,
StateSensor * newState) StateSensor * newState)
: throughputInKbps(0) : throughputInKbps(0)
, maxThroughput(0)
, packetHistory(newPacketHistory) , packetHistory(newPacketHistory)
, state(newState) , state(newState)
{ {
throughputInKbps = 0;
} }
int ThroughputSensor::getThroughputInKbps(void) const int ThroughputSensor::getThroughputInKbps(void) const
...@@ -35,8 +35,23 @@ void ThroughputSensor::localAck(PacketInfo * packet) ...@@ -35,8 +35,23 @@ void ThroughputSensor::localAck(PacketInfo * packet)
// period is in seconds. // period is in seconds.
double period = (currentAckTime - lastAckTime).toMilliseconds() / 1000.0; double period = (currentAckTime - lastAckTime).toMilliseconds() / 1000.0;
double kilobits = packetHistory->getAckedSize() * (8.0/1000.0); double kilobits = packetHistory->getAckedSize() * (8.0/1000.0);
throughputInKbps = static_cast<int>(kilobits/period); int latest = static_cast<int>(kilobits/period);
if (state->isSaturated() || latest > maxThroughput)
{
throughputInKbps = latest;
maxThroughput = latest;
logWrite(SENSOR, "THROUGHPUT: %d kbps", throughputInKbps);
}
}
else
{
throughputInKbps = 0;
} }
lastAckTime = currentAckTime; lastAckTime = currentAckTime;
} }
else
{
throughputInKbps = 0;
lastAckTime = Time();
}
} }
...@@ -20,6 +20,7 @@ protected: ...@@ -20,6 +20,7 @@ protected:
virtual void localAck(PacketInfo * packet); virtual void localAck(PacketInfo * packet);
private: private:
int throughputInKbps; int throughputInKbps;
int maxThroughput;
Time lastAckTime; Time lastAckTime;
PacketSensor * packetHistory; PacketSensor * packetHistory;
StateSensor * state; StateSensor * state;
......
...@@ -42,6 +42,7 @@ extern char * optarg; ...@@ -42,6 +42,7 @@ extern char * optarg;
#include <string> #include <string>
#include <sstream> #include <sstream>
#include <algorithm> #include <algorithm>
#include <iomanip>
#include "Time.h" #include "Time.h"
...@@ -89,7 +90,8 @@ enum SensorType ...@@ -89,7 +90,8 @@ enum SensorType
DELAY_SENSOR = 3, DELAY_SENSOR = 3,
MIN_DELAY_SENSOR = 4, MIN_DELAY_SENSOR = 4,
MAX_DELAY_SENSOR = 5, MAX_DELAY_SENSOR = 5,
THROUGHPUT_SENSOR = 6 THROUGHPUT_SENSOR = 6,
EWMA_THROUGHPUT_SENSOR = 7
}; };
// This is used for the type field in the ConnectionModelCommand. // This is used for the type field in the ConnectionModelCommand.
...@@ -143,7 +145,7 @@ struct Order ...@@ -143,7 +145,7 @@ struct Order
{ {
return !(*this == right); return !(*this == right);
} }
std::string toString(void) std::string toString(void) const
{ {
std::ostringstream buffer; std::ostringstream buffer;
if (transport == TCP_CONNECTION) if (transport == TCP_CONNECTION)
......
...@@ -70,6 +70,10 @@ static void logPrefix(int flags) ...@@ -70,6 +70,10 @@ static void logPrefix(int flags)
{ {
fprintf(logFile, "PCAP "); fprintf(logFile, "PCAP ");
} }
if (flags & COMMAND_OUTPUT)
{
fprintf(logFile, "COMMAND_OUTPUT ");
}
if (logTimestamp) if (logTimestamp)
{ {
struct timeval now; struct timeval now;
......
...@@ -48,9 +48,10 @@ enum LOG_TYPE ...@@ -48,9 +48,10 @@ enum LOG_TYPE
COMMAND_INPUT = 0x080, COMMAND_INPUT = 0x080,
CONNECTION = 0x100, CONNECTION = 0x100,
PCAP = 0x200, PCAP = 0x200,
COMMAND_OUTPUT = 0x400,
// Shortcuts for common cases. // Shortcuts for common cases.
LOG_NOTHING = 0x000, LOG_NOTHING = 0x000,
LOG_EVERYTHING = 0x3ff LOG_EVERYTHING = 0x7ff
}; };
#endif #endif
...@@ -260,8 +260,16 @@ void init(void) ...@@ -260,8 +260,16 @@ void init(void)
"Terminating program.", global::connectionModelArg); "Terminating program.", global::connectionModelArg);
} }
global::input.reset(new DirectInput()); if (global::replayArg == REPLAY_LOAD)
global::output.reset(new TrivialCommandOutput()); {
global::input.reset(new NullCommandInput());
global::output.reset(new NullCommandOutput());
}
else
{
global::input.reset(new DirectInput());
global::output.reset(new TrivialCommandOutput());
}
} }
void mainLoop(void) void mainLoop(void)
...@@ -606,6 +614,17 @@ int createServer(int port, string const & debugString) ...@@ -606,6 +614,17 @@ int createServer(int port, string const & debugString)
return -1; return -1;
} }
int forcedSize = 262144;
error = setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &forcedSize,
sizeof(forcedSize));
if (error == -1)
{
logWrite(ERROR, "Failed to set receive buffer size: %s",
strerror(errno));
close(fd);
return -1;
}
struct sockaddr_in address; struct sockaddr_in address;
address.sin_family = AF_INET; address.sin_family = AF_INET;
address.sin_port = htons(port); address.sin_port = htons(port);
......
...@@ -32,6 +32,7 @@ DELAY_SENSOR = 3 ...@@ -32,6 +32,7 @@ DELAY_SENSOR = 3
MIN_DELAY_SENSOR = 4 MIN_DELAY_SENSOR = 4
MAX_DELAY_SENSOR = 5 MAX_DELAY_SENSOR = 5
THROUGHPUT_SENSOR = 6 THROUGHPUT_SENSOR = 6
EWMA_THROUGHPUT_SENSOR = 7
CONNECTION_SEND_BUFFER_SIZE = 0 CONNECTION_SEND_BUFFER_SIZE = 0
CONNECTION_RECEIVE_BUFFER_SIZE = 1 CONNECTION_RECEIVE_BUFFER_SIZE = 1
...@@ -185,7 +186,11 @@ def get_next_packet(conn): ...@@ -185,7 +186,11 @@ def get_next_packet(conn):
send_command(conn, SENSOR_COMMAND, TCP_CONNECTION, ipaddr, send_command(conn, SENSOR_COMMAND, TCP_CONNECTION, ipaddr,
localport, remoteport, save_int(MIN_DELAY_SENSOR)) localport, remoteport, save_int(MIN_DELAY_SENSOR))
send_command(conn, SENSOR_COMMAND, TCP_CONNECTION, ipaddr, send_command(conn, SENSOR_COMMAND, TCP_CONNECTION, ipaddr,
localport, remoteport, save_int(NULL_SENSOR)) # localport, remoteport, save_int(NULL_SENSOR))
# send_command(conn, SENSOR_COMMAND, TCP_CONNECTION, ipaddr,
localport, remoteport, save_int(MAX_DELAY_SENSOR))
send_command(conn, SENSOR_COMMAND, TCP_CONNECTION, ipaddr,
localport, remoteport, save_int(EWMA_THROUGHPUT_SENSOR))
elif event == 'Closed': elif event == 'Closed':
send_command(conn, DELETE_CONNECTION_COMMAND, TCP_CONNECTION, ipaddr, send_command(conn, DELETE_CONNECTION_COMMAND, TCP_CONNECTION, ipaddr,
localport, remoteport, '') localport, remoteport, '')
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment