netmap-net-device-helper.cc

Go to the documentation of this file.

/*
 * Copyright (c) 2017 Universita' degli Studi di Napoli Federico II
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation;
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *
 * Author: Pasquale Imputato <p.imputato@gmail.com>
 */
 
#include "netmap-net-device-helper.h"
 
#include "encode-decode.h"
 
#include "ns3/abort.h"
#include "ns3/config.h"
#include "ns3/fd-net-device.h"
#include "ns3/log.h"
#include "ns3/names.h"
#include "ns3/netmap-net-device.h"
#include "ns3/object-factory.h"
#include "ns3/packet.h"
#include "ns3/simulator.h"
#include "ns3/trace-helper.h"
#include "ns3/uinteger.h"
 
#include <arpa/inet.h>
#include <errno.h>
#include <iomanip>
#include <iostream>
#include <limits>
#include <memory>
#include <net/ethernet.h>
#include <net/if.h>
#include <net/netmap_user.h>
#include <netinet/in.h>
#include <stdlib.h>
#include <string.h>
#include <string>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/un.h>
#include <sys/wait.h>
#include <time.h>
#include <unistd.h>
 
namespace ns3
{
 
NS_LOG_COMPONENT_DEFINE("NetmapNetDeviceHelper");
 
#define EMU_MAGIC 65867
 
NetmapNetDeviceHelper::NetmapNetDeviceHelper()
{
    m_deviceName = "undefined";
    SetTypeId("ns3::NetmapNetDevice");
}
 
std::string
NetmapNetDeviceHelper::GetDeviceName()
{
    return m_deviceName;
}
 
void
NetmapNetDeviceHelper::SetDeviceName(std::string deviceName)
{
    m_deviceName = deviceName;
}
 
Ptr<NetDevice>
NetmapNetDeviceHelper::InstallPriv(Ptr<Node> node) const
{
    Ptr<NetDevice> d = FdNetDeviceHelper::InstallPriv(node);
    Ptr<FdNetDevice> device = d->GetObject<FdNetDevice>();
 
    SetDeviceAttributes(device);
 
    int fd = CreateFileDescriptor();
    Ptr<NetmapNetDevice> netmapDevice = DynamicCast<NetmapNetDevice>(device);
    SwitchInNetmapMode(fd, netmapDevice);
 
    // Aggregate NetDeviceQueueInterface object
    Ptr<NetDeviceQueueInterface> ndqi = CreateObjectWithAttributes<NetDeviceQueueInterface>(
        "TxQueuesType",
        TypeIdValue(NetDeviceQueueLock::GetTypeId()),
        "NTxQueues",
        UintegerValue(1));
 
    device->AggregateObject(ndqi);
    netmapDevice->SetNetDeviceQueue(ndqi->GetTxQueue(0));
 
    return device;
}
 
void
NetmapNetDeviceHelper::SetDeviceAttributes(Ptr<FdNetDevice> device) const
{
    if (m_deviceName == "undefined")
    {
        NS_FATAL_ERROR("NetmapNetDeviceHelper::SetFileDescriptor (): m_deviceName is not set");
    }
 
    //
    // Call out to a separate process running as suid root in order to get a raw
    // socket.  We do this to avoid having the entire simulation running as root.
    //
    int fd = socket(PF_INET, SOCK_DGRAM, 0);
 
    //
    // Figure out which interface index corresponds to the device name in the corresponding
    // attribute.
    //
    struct ifreq ifr;
    bzero(&ifr, sizeof(ifr));
    strncpy((char*)ifr.ifr_name, m_deviceName.c_str(), IFNAMSIZ - 1);
 
    NS_LOG_LOGIC("Getting interface index");
    int32_t rc = ioctl(fd, SIOCGIFINDEX, &ifr);
    if (rc == -1)
    {
        NS_FATAL_ERROR("NetmapNetDeviceHelper::SetFileDescriptor (): Can't get interface index");
    }
 
    rc = ioctl(fd, SIOCGIFFLAGS, &ifr);
    if (rc == -1)
    {
        NS_FATAL_ERROR("NetmapNetDeviceHelper::SetFileDescriptor (): Can't get interface flags");
    }
 
    //
    // This device only works if the underlying interface is up in promiscuous
    // mode.  We could have turned it on in the socket creator, but the situation
    // is that we expect these devices to be used in conjunction with virtual
    // machines with connected host-only (simulated) networks, or in a testbed.
    // There is a lot of setup and configuration happening outside of this one
    // issue, and we expect that configuration to include choosing a valid
    // interface (e.g, "ath1"), ensuring that the device supports promiscuous
    // mode, and placing it in promiscuous mode.  We just make sure of the
    // end result.
    //
    if ((ifr.ifr_flags & IFF_PROMISC) == 0)
    {
        NS_FATAL_ERROR("NetmapNetDeviceHelper::SetFileDescriptor (): "
                       << m_deviceName
                       << " is not in promiscuous mode. Please config the interface in promiscuous "
                          "mode before to run the simulation.");
    }
 
    if ((ifr.ifr_flags & IFF_BROADCAST) != IFF_BROADCAST)
    {
        // We default m_isBroadcast to true but turn it off here if not
        // supported, because in the common case, overlying IP code will
        // assert during configuration time if this is false, before this
        // method has a chance to set it during runtime
        device->SetIsBroadcast(false);
    }
 
    if ((ifr.ifr_flags & IFF_MULTICAST) == IFF_MULTICAST)
    {
        // This one is OK to enable at runtime
        device->SetIsMulticast(true);
    }
 
    // Set the MTU of the device to the mtu of the associated network interface
    //   struct ifreq ifr2;
    //
    //   bzero (&ifr2, sizeof (ifr2));
    //   strcpy (ifr2.ifr_name, m_deviceName.c_str ());
 
    //   int32_t mtufd = socket (PF_INET, SOCK_DGRAM, IPPROTO_IP);
 
    rc = ioctl(fd, SIOCGIFMTU, &ifr);
    if (rc == -1)
    {
        NS_FATAL_ERROR("FdNetDevice::SetFileDescriptor (): Can't ioctl SIOCGIFMTU");
    }
 
    NS_LOG_DEBUG("Device MTU " << ifr.ifr_mtu);
    device->SetMtu(ifr.ifr_mtu);
 
    close(fd);
}
 
int
NetmapNetDeviceHelper::CreateFileDescriptor() const
{
    NS_LOG_FUNCTION(this);
 
    //
    // We want to create a raw socket for our net device.  Unfortunately for us
    // you have to have root privileges to do that.  Instead of running the
    // entire simulation as root, we decided to make a small program who's whole
    // reason for being is to run as suid root and create a raw socket.  We're
    // going to fork and exec that program soon, but we need to have a socket
    // to talk to it with.  So we create a local interprocess (Unix) socket
    // for that purpose.
    //
    int sock = socket(PF_UNIX, SOCK_DGRAM, 0);
    if (sock == -1)
    {
        NS_FATAL_ERROR(
            "NetmapNetDeviceHelper::CreateFileDescriptor(): Unix socket creation error, errno = "
            << strerror(errno));
    }
 
    //
    // Bind to that socket and let the kernel allocate an endpoint
    //
    struct sockaddr_un un;
    memset(&un, 0, sizeof(un));
    un.sun_family = AF_UNIX;
    int status = bind(sock, (struct sockaddr*)&un, sizeof(sa_family_t));
    if (status == -1)
    {
        NS_FATAL_ERROR("NetmapNetDeviceHelper::CreateFileDescriptor(): Could not bind(): errno = "
                       << strerror(errno));
    }
 
    NS_LOG_INFO("Created Unix socket");
    NS_LOG_INFO("sun_family = " << un.sun_family);
    NS_LOG_INFO("sun_path = " << un.sun_path);
 
    //
    // We have a socket here, but we want to get it there -- to the program we're
    // going to exec.  What we'll do is to do a getsockname and then encode the
    // resulting address information as a string, and then send the string to the
    // program as an argument.  So we need to get the sock name.
    //
    socklen_t len = sizeof(un);
    status = getsockname(sock, (struct sockaddr*)&un, &len);
    if (status == -1)
    {
        NS_FATAL_ERROR(
            "NetmapNetDeviceHelper::CreateFileDescriptor(): Could not getsockname(): errno = "
            << strerror(errno));
    }
 
    //
    // Now encode that socket name (family and path) as a string of hex digits
    //
    std::string path = BufferToString((uint8_t*)&un, len);
    NS_LOG_INFO("Encoded Unix socket as \"" << path << "\"");
    //
    // Fork and exec the process to create our socket.  If we're us (the parent)
    // we wait for the child (the socket creator) to complete and read the
    // socket it created using the ancillary data mechanism.
    //
    // Tom Goff reports the possibility of a deadlock when trying to acquire the
    // python GIL here.  He says that this might be due to trying to access Python
    // objects after fork() without calling PyOS_AfterFork() to properly reset
    // Python state (including the GIL).  There is no code to cause the problem
    // here in emu, but this was visible in similar code in tap-bridge.
    //
    pid_t pid = ::fork();
    if (pid == 0)
    {
        NS_LOG_DEBUG("Child process");
 
        //
        // build a command line argument from the encoded endpoint string that
        // the socket creation process will use to figure out how to respond to
        // the (now) parent process.
        //
        std::ostringstream oss;
 
        oss << "-p" << path;
 
        NS_LOG_INFO("Parameters set to \"" << oss.str() << "\"");
 
        //
        // Execute the socket creation process image.
        //
        status = ::execlp(NETMAP_DEV_CREATOR,
                          NETMAP_DEV_CREATOR, // argv[0] (filename)
                          oss.str().c_str(),  // argv[1] (-p<path?
                          nullptr);
 
        //
        // If the execlp successfully completes, it never returns.  If it returns it failed or the
        // OS is broken.  In either case, we bail.
        //
        NS_FATAL_ERROR(
            "NetmapNetDeviceHelper::CreateFileDescriptor(): Back from execlp(), status = "
            << status << ", errno = " << ::strerror(errno));
    }
    else
    {
        NS_LOG_DEBUG("Parent process");
        //
        // We're the process running the emu net device.  We need to wait for the
        // socket creator process to finish its job.
        //
        int st;
        pid_t waited = waitpid(pid, &st, 0);
        if (waited == -1)
        {
            NS_FATAL_ERROR(
                "NetmapNetDeviceHelper::CreateFileDescriptor(): waitpid() fails, errno = "
                << strerror(errno));
        }
        NS_ASSERT_MSG(pid == waited, "NetmapNetDeviceHelper::CreateFileDescriptor(): pid mismatch");
 
        //
        // Check to see if the socket creator exited normally and then take a
        // look at the exit code.  If it bailed, so should we.  If it didn't
        // even exit normally, we bail too.
        //
        if (WIFEXITED(st))
        {
            int exitStatus = WEXITSTATUS(st);
            if (exitStatus != 0)
            {
                NS_FATAL_ERROR("NetmapNetDeviceHelper::CreateFileDescriptor(): socket creator "
                               "exited normally with status "
                               << exitStatus);
            }
        }
        else
        {
            NS_FATAL_ERROR(
                "NetmapNetDeviceHelper::CreateFileDescriptor(): socket creator exited abnormally");
        }
 
        //
        // At this point, the socket creator has run successfully and should
        // have created our raw socket and sent it back to the socket address
        // we provided.  Our socket should be waiting on the Unix socket.  We've
        // got to do a bunch of grunto work to get at it, though.
        //
        // The struct iovec below is part of a scatter-gather list.  It describes a
        // buffer.  In this case, it describes a buffer (an integer) that will
        // get the data that comes back from the socket creator process.  It will
        // be a magic number that we use as a consistency/sanity check.
        //
        struct iovec iov;
        uint32_t magic;
        iov.iov_base = &magic;
        iov.iov_len = sizeof(magic);
 
        //
        // The CMSG macros you'll see below are used to create and access control
        // messages (which is another name for ancillary data).  The ancillary
        // data is made up of pairs of struct cmsghdr structures and associated
        // data arrays.
        //
        // First, we're going to allocate a buffer on the stack to receive our
        // data array (that contains the socket).  Sometimes you'll see this called
        // an "ancillary element" but the msghdr uses the control message terminology
        // so we call it "control."
        //
        size_t msg_size = sizeof(int);
        char control[CMSG_SPACE(msg_size)];
 
        //
        // There is a msghdr that is used to minimize the number of parameters
        // passed to recvmsg (which we will use to receive our ancillary data).
        // This structure uses terminology corresponding to control messages, so
        // you'll see msg_control, which is the pointer to the ancillary data and
        // controller which is the size of the ancillary data array.
        //
        // So, initialize the message header that describes the ancillary/control
        // data we expect to receive and point it to buffer.
        //
        struct msghdr msg;
        msg.msg_name = nullptr;
        msg.msg_namelen = 0;
        msg.msg_iov = &iov;
        msg.msg_iovlen = 1;
        msg.msg_control = control;
        msg.msg_controllen = sizeof(control);
        msg.msg_flags = 0;
 
        //
        // Now we can actually receive the interesting bits from the socket
        // creator process.
        //
        ssize_t bytesRead = recvmsg(sock, &msg, 0);
        if (bytesRead != sizeof(int))
        {
            NS_FATAL_ERROR("NetmapNetDeviceHelper::CreateFileDescriptor(): Wrong byte count from "
                           "socket creator");
        }
 
        //
        // There may be a number of message headers/ancillary data arrays coming in.
        // Let's look for the one with a type SCM_RIGHTS which indicates it' the
        // one we're interested in.
        //
        struct cmsghdr* cmsg;
        for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != nullptr; cmsg = CMSG_NXTHDR(&msg, cmsg))
        {
            if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS)
            {
                //
                // This is the type of message we want.  Check to see if the magic
                // number is correct and then pull out the socket we care about if
                // it matches
                //
                if (magic == EMU_MAGIC)
                {
                    NS_LOG_INFO("Got SCM_RIGHTS with correct magic " << magic);
                    int* rawSocket = (int*)CMSG_DATA(cmsg);
                    NS_LOG_INFO("Got the socket from the socket creator = " << *rawSocket);
                    return *rawSocket;
                }
                else
                {
                    NS_LOG_INFO("Got SCM_RIGHTS, but with bad magic " << magic);
                }
            }
        }
        NS_FATAL_ERROR("Did not get the raw socket from the socket creator");
    }
}
 
void
NetmapNetDeviceHelper::SwitchInNetmapMode(int fd, Ptr<NetmapNetDevice> device) const
{
    NS_LOG_FUNCTION(this << fd << device);
    NS_ASSERT(device);
 
    if (m_deviceName == "undefined")
    {
        NS_FATAL_ERROR("NetmapNetDevice: m_deviceName is not set");
    }
 
    if (fd == -1)
    {
        NS_FATAL_ERROR("NetmapNetDevice: fd is not set");
    }
 
    struct nmreq nmr;
    memset(&nmr, 0, sizeof(nmr));
 
    nmr.nr_version = NETMAP_API;
 
    // setting the interface name in the netmap request
    strncpy(nmr.nr_name, m_deviceName.c_str(), m_deviceName.length());
 
    // switch the interface in netmap mode
    int code = ioctl(fd, NIOCREGIF, &nmr);
    if (code == -1)
    {
        NS_FATAL_ERROR("Switching failed");
    }
 
    // memory mapping
    uint8_t* memory = (uint8_t*)mmap(0, nmr.nr_memsize, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0);
 
    if (memory == MAP_FAILED)
    {
        NS_FATAL_ERROR("Memory mapping failed");
    }
 
    // getting the base struct of the interface in netmap mode
    struct netmap_if* nifp = NETMAP_IF(memory, nmr.nr_offset);
 
    if (!nifp)
    {
        NS_FATAL_ERROR("Failed getting the base struct of the interface in netmap mode");
    }
 
    device->SetNetmapInterfaceRepresentation(nifp);
    device->SetTxRingsInfo(nifp->ni_tx_rings, nmr.nr_tx_slots);
    device->SetRxRingsInfo(nifp->ni_rx_rings, nmr.nr_rx_slots);
 
    device->SetFileDescriptor(fd);
}
 
} // namespace ns3