windows-server-2003/base/cluster/service/gs/msg.c

/*++

Copyright (c) 2000  Microsoft Corporation

Module Name:

    msg.c

Abstract:

    Point to point tcp and ip-multicast

Author:

    Ahmed Mohamed (ahmedm) 12, 01, 2000

Revision History:

--*/

#include <stdio.h>
#include "msg.h"


#include <stdlib.h>
#include <winsock2.h>
#include <ctype.h>
#include <string.h>
#include <assert.h>
#include <time.h>

#define MSG_ATOMIC 1

int GS_MAX_MSG_SZ = (64 * 1024);

#define PROTOCOL_TYPE	SOCK_STREAM

extern gs_node_handler_t gs_node_handler[];

static int max_mcmsg = 0;

static char cl_subnet[16];

#define MAX_NODEID	16

int NodesSize = 0;

static char *nodes[MAX_NODEID] = {0};
static char *ipaddr[MAX_NODEID] = {0};

static int DEFAULT_PORT=6009;

static int mcast_enabled = 0;
static int MSG_POOL_SZ=32;

int MY_NODEID;

static SOCKET	prf_handles[MAX_NODEID];
static SOCKET	rcv_handles[MAX_NODEID];
static SOCKET	send_handles[MAX_NODEID];
static SOCKET	tmp_socks[MAX_NODEID];

static CRITICAL_SECTION msglock;
static HANDLE Msg_Event[MAX_NODEID];

void mcast_init();
DWORD WINAPI srv(LPVOID arg);
DWORD WINAPI mcast_srv(LPVOID arg);
DWORD WINAPI srv_io(LPVOID arg);
DWORD WINAPI cmgr(LPVOID arg);

static gs_msg_t *msg_pool = NULL;
static gs_msg_t *msg_hdrpool = NULL;


void
Msg_AllocPool()
{
    char *p;
    gs_msg_t *prev;
    int sz, elmsz;

    // allocate msg header pool
    sz = sizeof(gs_msg_t) * MSG_POOL_SZ;
    prev = (gs_msg_t *) malloc(sz);
    if (prev == NULL) {
	printf("Unable to allocate message hdr pool\n");
	halt(1);
    }
    msg_hdrpool = NULL;
    for (sz = 0; sz < MSG_POOL_SZ; sz++) {
	prev->m_refcnt = 0;
	prev->m_buflen = 0;
	prev->m_buf = NULL;
	prev->m_next = msg_hdrpool;
	msg_hdrpool = prev;
	prev++;
    }

    // allocate msg pool
    sz = sizeof(gs_msg_t) * MSG_POOL_SZ;
//    prev = (gs_msg_t *) malloc(sz);
    prev = (gs_msg_t *) VirtualAlloc(NULL, sz, MEM_RESERVE|MEM_COMMIT,
		     PAGE_READWRITE);
    if (prev == NULL) {
	printf("Unable to allocate message pool\n");
	halt(1);
    }

    // lock region now
    if (!VirtualLock(prev, sz)) {
	printf("Unable to lock down hdr pages %d\n", GetLastError());
    }

    sz = MSG_POOL_SZ * GS_MAX_MSG_SZ;
    p = VirtualAlloc(NULL, sz, MEM_RESERVE|MEM_COMMIT,
		     PAGE_READWRITE);
    if (p == NULL) {
	printf("Unable to allocate message memory pool\n");
	halt(1);
    }

    if (!VirtualLock(p, sz)) {
	printf("Unable to lock down pages %d err %d\n", sz, GetLastError());
    }

    msg_pool = NULL;
    for (sz = 0; sz < MSG_POOL_SZ; sz++) {
	prev->m_refcnt = 0;
	prev->m_buflen = GS_MAX_MSG_SZ - 1;
	prev->m_buf = p;
	prev->m_next = msg_pool;
	msg_pool = prev;
	prev++;
	*p = 0;	// touch it
	p += GS_MAX_MSG_SZ;
    }

}


gs_msg_t *
msg_hdralloc(const char *buf, int len)
{
    PVOID t;
    gs_msg_t * p;

#ifdef MSG_ATOMIC
    do {
	p = msg_hdrpool;
	if (p == NULL) {
	    break;
	}
	t = InterlockedCompareExchangePointer((PVOID *)&msg_hdrpool, 
					      (PVOID)p->m_next, (PVOID) p);
    } while (t != (PVOID) p);
#else
    GsLockEnter(msglock);
    if (p = msg_hdrpool) {
	msg_hdrpool = p->m_next;
	p->m_next = NULL;
    }
    GsLockExit(msglock);
#endif    

    if (p == NULL) {
	printf("Out of message headers!!!\n");
	halt(1);
    }
    
//    p->m_buflen = 0;
    p->m_refcnt = 1;
    p->m_buf = (char *)buf;
    p->m_type = MSG_TYPE_HDR;

    msg_log(("Alloc hdr msg %x len %d pool %x\n", p, p->m_buflen, msg_hdrpool));
    return p;

}

gs_msg_t *
msg_alloc(const char *buf, int len)
{
    PVOID t;
    gs_msg_t * p;

    if (len > GS_MAX_MSG_SZ) {
	printf("Large msg, can't handle %d\n", len);
	halt(1);
    }

    if (buf != NULL) {
	return msg_hdralloc(buf, len);
    }

#ifdef MSG_ATOMIC
    do {
	p = msg_pool;
	if (p == NULL) {
	    break;
	}
	t = InterlockedCompareExchangePointer((PVOID *)&msg_pool, 
					      (PVOID)p->m_next, (PVOID) p);
    } while (t != (PVOID) p);
#else
    GsLockEnter(msglock);
    if (p = msg_pool) {
	msg_pool = p->m_next;
	p->m_next = NULL;
	msg_log(("Alloc msg %x pool %x\n", p, msg_pool));
    }
    GsLockExit(msglock);
#endif    

    if (p == NULL) {
	printf("Out of messages!!!\n");
	halt(1);
    }
    
//    p->m_buflen = len;
    p->m_refcnt = 1;
    p->m_type = MSG_TYPE_DATA;

    if (buf) {
	memcpy(p->m_buf, buf, len);
    }

    msg_log(("Alloc msg %x buf %x len %d\n", p, p->m_buf, p->m_buflen));

    return p;

}


void
msg_hdrfree(gs_msg_t *msg)
{
    PVOID t, p;
    
    msg_log(("Free hdr msg %x len %d pool %x\n", msg, msg->m_buflen,msg_pool));
#ifdef MSG_ATOMIC
    do {
	msg->m_next = msg_hdrpool;
	t = InterlockedCompareExchangePointer((PVOID *)&msg_hdrpool, (PVOID)msg, 
					      (PVOID)msg->m_next);
    } while (t != (PVOID) msg->m_next);
#else
    GsLockEnter(msglock);
    msg->m_next = msg_hdrpool;
    msg_hdrpool = msg;
    GsLockExit(msglock);
#endif
}

void
msg_free(gs_msg_t *msg)
{
    PVOID t, p;
    
    msg->m_refcnt--;
    if (msg->m_refcnt > 0) {
	msg_log(("msg %x not freed %d flags %x\n", msg, msg->m_refcnt, 
		 msg->m_hdr.h_flags));
	if (msg->m_refcnt > 10) {
	    halt(0);
	}
	return;
    }
    if (msg->m_type == MSG_TYPE_HDR) {
	msg_hdrfree(msg);
	return;
    }
    msg_log(("Free msg %x buf %x pool %x\n", msg, msg->m_buf, msg_pool));
#ifdef MSG_ATOMIC
    do {
	msg->m_next = msg_pool;
	t = InterlockedCompareExchangePointer((PVOID *)&msg_pool, (PVOID)msg, 
					      (PVOID)msg->m_next);
    } while (t != (PVOID) msg->m_next);
#else
    GsLockEnter(msglock);
    msg->m_next = msg_pool;
    msg_pool = msg;
    GsLockExit(msglock);
#endif
}


char *
strsave(char *s)
{
  char *p;

  p = (char*)malloc(strlen(s) + 1);
  assert(p != NULL);
  return strcpy(p, s);
}

int 
Strncasecmp(char *s, char *p, int len)
{
  while (len-- > 0) {
    if (tolower(s[len]) != tolower(p[len]))
      return 1;
  }
  return 0;
}

/********************************************************************/

int
msg_buildaddr(struct sockaddr_in *sin, char *hostname, char *ipaddr)
{
    
    struct hostent *h;
    int i;
    char *p;
    char *tmp;

    h = gethostbyname(hostname);
    if (h == NULL) {
	fprintf(stderr,"cannot get info for host %s\n", hostname);
	return 1;
    }

    p = (char *) h->h_addr_list[0];
    for (i = 0; h->h_addr_list[i]; i++) {
	struct in_addr x;

	memcpy(&x, p, h->h_length);
	tmp = inet_ntoa(x);
	if (!strncmp(cl_subnet, tmp, strlen(cl_subnet))) {
	    break;
	}
	p += h->h_length;
    }
    if (h->h_addr_list[i] == NULL) {
	printf("Unable to find proper subnet %s host %s\n", cl_subnet, hostname);
	if (ipaddr != NULL) {
	    // use this address
	    sin->sin_addr.s_addr = inet_addr(ipaddr);
	    printf("host %s addr %s\n", hostname, ipaddr);
	} else {
	    sin->sin_addr.s_addr = INADDR_ANY;
	    printf("host %s addr %s\n", hostname, "any");
	}
    } else {
	memcpy(&sin->sin_addr.s_addr, h->h_addr_list[i], h->h_length);
	printf("host %s addr %s\n", hostname, tmp);
    }
    return 0;
    
}


#ifndef TEST
int
WINAPI
msg_addnode(int id, char *n, char *a)
{
    char *s;

    s = strchr(n, '.');	
    if (s) {
	*s = '\0';
    }

    printf("nodeid %d node %s ip %s\n", id, n, a);


    nodes[id-1] = strsave(n);
    ipaddr[id-1] = strsave(a);

    if (id > NodesSize)
	NodesSize = id;

    return NodesSize;

}
#endif
/********************************************************************/

int
msg_getsize()
{
    return NodesSize;
}

void
msg_closenode(int nodeid)
{

    GsLockEnter(msglock);
    if (rcv_handles[nodeid]) {
	closesocket(rcv_handles[nodeid]);
	rcv_handles[nodeid] = 0;
    }
    if (send_handles[nodeid]) {
	closesocket(send_handles[nodeid]);
	send_handles[nodeid] = 0;
    }
    prf_handles[nodeid] = 0;
    GsLockExit(msglock);
    return;
}

int
msg_send(gs_memberid_t nodeid, gs_msg_hdr_t *hdr, const char *buf, int len)
{
	int i;
	SOCKET s;
	WSABUF	io[2];
	LPWSAOVERLAPPED ov;

	ov = NULL;

	nodeid--;
	if (nodeid >= NodesSize) {
	    err_log(("send bad node %d\n", nodeid));
	    return 1;
	}

	s = prf_handles[nodeid];
	if (!s) {
	    s = send_handles[nodeid];
	    if (!s) {
		s = rcv_handles[nodeid];
		if (!s) {
		    err_log(("Node %d is dead\n", nodeid+1));
		    return 1;
		}
	    }
	}

	msg_log(("Send msg nid %d type %d seq %d bnum %d view %d\n",
		 nodeid+1, hdr->h_type, hdr->h_mseq,
		 hdr->h_bnum, hdr->h_viewnum));
	io[0].len = sizeof(*hdr);
	io[0].buf = (char *) hdr;
	io[1].len = len;
	io[1].buf = (char *) buf;

	if (WSASend(s, io, 2, &i, 0, ov, 0)) {
	    int err = WSAGetLastError();
	    if (err == WSA_IO_PENDING) {
		printf("Async send\n");
		return 1;
	    }
	    printf("Send nid %d failed %d\n", nodeid+1, err);
	    msg_closenode(nodeid);
	    return 1;
	}
	i -= sizeof(*hdr);
	if (i != len) {
	    printf("Send failed: node %d len %d, %d\n", nodeid+1, len, i);
	    halt(1);
	}
	return 0;
}


void
msg_mcast(ULONG mset, gs_msg_hdr_t *hdr, const char *buf, int len)
{
    gs_memberid_t i;
    void mcast_send(gs_msg_hdr_t *hdr, const char *buf, int len);

    mset = mset & ~(1 << MY_NODEID);
    if (mset == 0)
	return;
//    if (mcast_enabled == 0 || len > max_mcmsg) {
    if (len > max_mcmsg) {
	
	for (i = 1; i <= NodesSize; i++) {
	    if (mset & (1 << i)) {
		msg_send(i, hdr, buf, len);
	    }
	}
    }
    else {
	mcast_send(hdr, buf, len);
    }
}

void
msg_smcast(ULONG mset, gs_msg_hdr_t *hdr, const char *buf, int len)
{
    gs_memberid_t i;

    mset = mset & ~(1 << MY_NODEID);
    if (mset == 0)
	return;
    for (i = 1; i <= NodesSize; i++) {
	if (mset & (1 << i)) {
	    msg_send(i, hdr, buf, len);
	}
    }
}

msg_init()
{
	int i;
	WSADATA wsaData;
	char h_name[64];

	// set our priority to high class
	if (!SetPriorityClass(GetCurrentProcess(),HIGH_PRIORITY_CLASS)) {
	    printf("Unable to set high priority %d\n", GetLastError());
	}

	if (WSAStartup(0x202,&wsaData) == SOCKET_ERROR) {
		fprintf(stderr,"WSAStartup failed with error %d\n",
			WSAGetLastError());
		WSACleanup();
		return -1;
	}

	i = gethostname(h_name, 64);


	// increase our working set
	if (!SetProcessWorkingSetSize(GetCurrentProcess(),
				      32*1024*1024, 64*1024*1024)) {
	    printf("Unable to set working size %d\n", GetLastError());
	}

	InitializeCriticalSection(&msglock);

	Msg_AllocPool();


	for (i = 0; i < NodesSize; i++) {
	    Msg_Event[i] = CreateEvent(NULL, TRUE, FALSE, NULL);
	    prf_handles[i] = 0;
	    send_handles[i] = 0;
	    rcv_handles[i] = 0;
	    if (!Strncasecmp(h_name, nodes[i], strlen(h_name))) {
		MY_NODEID = i+1;
		gs_node_handler[MSG_NODE_ID](MY_NODEID);
	    } else {
		LPVOID arg = (LPVOID) ((ULONGLONG) i);
		CreateThread(NULL, 2*64*1024, cmgr, arg, 0, NULL);
	    }
	}
	cm_log(("Local host %d %s\n", MY_NODEID, h_name));

	if (mcast_enabled) {
	    mcast_init();
	}

	if (NodesSize > 1) {
	    LPVOID arg = (LPVOID) ((ULONGLONG) DEFAULT_PORT);
	    // create srv thread
	    CreateThread(NULL, 4*1024, srv, arg, 0,NULL);
	    // create mcast thread
	    if (mcast_enabled) {
		for (i = 0; i < 8; i++)
		    CreateThread(NULL, 2*64*1024, mcast_srv, 0, 0,NULL);
	    }
	}

	return 0;
}

void
msg_exit()
{
    // xxx: Stop all threads before during this
    WSACleanup();
}

void
msg_start(ULONG mask)
{
    int i;

    mask = mask >> 1;
    for (i = 0; i < NodesSize; i++) {
	GsLockEnter(msglock);
	if (!(mask & (1 << i)) && !send_handles[i]) {
	    SetEvent(Msg_Event[i]);
	}
	GsLockExit(msglock);
    }
}
    
DWORD
srv_msg(SOCKET msgsock, int nodeid)
{
  gs_msg_t *msg;
  int retval;
  char *buf;
  int len;

  while (1) {
      extern gs_msg_handler_t gs_msg_handler[];
      int type;

      msg = msg_alloc(NULL, GS_MAX_MSG_SZ);

      // read hdr info first
      buf = (char *) &msg->m_hdr;
      len = sizeof(msg->m_hdr);

      do {
	  retval = recv(msgsock, buf, len, 0);
	  if (retval < 0) {
	      err_log(("recv failed %d, %d\n",
		       retval,
		       WSAGetLastError()));
	      msg_free(msg);
	      return 0;
	  }

	  len -= retval;
	  buf += retval;
      } while (len > 0);

      // read rest of message
      buf = msg->m_buf;
      len = msg->m_hdr.h_len;

      while (len > 0) {

	  retval = recv(msgsock, buf, len, 0);
	  if (retval < 0) {
	      err_log(("recv failed %d, %d\n",retval,
		       WSAGetLastError()));	
	      msg_free(msg);
	      return 0;
	  }
	  len -= retval;
	  buf += retval;
      }

      // set preferred socket to use
      prf_handles[nodeid] = msgsock;

      msg_log(("rec nid %d gid %d type %d seq %d view %d len %d\n",
	       msg->m_hdr.h_sid,msg->m_hdr.h_gid, type = msg->m_hdr.h_type,
	       msg->m_hdr.h_mseq, msg->m_hdr.h_viewnum, msg->m_hdr.h_len));

      gs_msg_handler[msg->m_hdr.h_type](msg);

      msg_log(("Done Type %d\n", type));
  }

  return 0;
}
    
DWORD WINAPI
srv_io(LPVOID arg)
{
  int retval;
  char *buf;
  int len;
  ULONGLONG tmp = (ULONGLONG) arg;
  int nodeid = (int) tmp;
  SOCKET msgsock = tmp_socks[nodeid];


  GsLockEnter(msglock);

  gs_node_handler[MSG_NODE_JOIN](nodeid+1);
  rcv_handles[nodeid] = msgsock;

  // issue join callback
  if (!send_handles[nodeid]) {
      gs_node_handler[MSG_NODE_UP](nodeid+1);
      SetEvent(Msg_Event[nodeid]);
  }
  GsLockExit(msglock);

  srv_msg(msgsock, nodeid);

  cm_log(("Terminating connection with node %d\n", nodeid));

  msg_closenode(nodeid);
  gs_node_handler[MSG_NODE_DOWN](nodeid+1);
  return (0);
}

void
msg_setopt(SOCKET s)
{
    // set option keepalive
    BOOLEAN val = TRUE;
    if (setsockopt(s, IPPROTO_TCP, SO_KEEPALIVE, (char *)&val,
		   sizeof(val)) == SOCKET_ERROR) {
	fprintf(stderr,"Keepalive %d\n", WSAGetLastError());
    }

    // set option nodelay
    val = TRUE;
    if (setsockopt(s, IPPROTO_TCP, TCP_NODELAY, (char *)&val,
		   sizeof(val)) == SOCKET_ERROR) {
	fprintf(stderr,"No delay %d\n", WSAGetLastError());
    }

    // set option nolinger
    val = TRUE;
    if (setsockopt(s, SOL_SOCKET, SO_DONTLINGER, (char *)&val,
		   sizeof(val)) == SOCKET_ERROR) {
	fprintf(stderr,"No delay %d\n", WSAGetLastError());
    }
}


DWORD WINAPI
srv(LPVOID arg)
{
	char *nic= NULL;
	int fromlen;
	int i;
	struct sockaddr_in local, from;
	SOCKET listen_socket, msgsock;
	ULONGLONG tmp = (ULONGLONG) arg;
	short port = (short) tmp;
#if 0
	nic = ipaddr[MY_NODEID-1];

	local.sin_addr.s_addr = (!nic)?INADDR_ANY:inet_addr(nic); 
#else
	if (msg_buildaddr(&local, nodes[MY_NODEID-1], ipaddr[MY_NODEID-1])) {
	    fprintf(stderr,"Unable to get my own address\n");
	    return -1;
	}
#endif
	local.sin_family = AF_INET;

	/* 
	 * Port MUST be in Network Byte Order
	 */
	local.sin_port = htons(port);

	// TCP socket
	listen_socket = WSASocket(AF_INET, PROTOCOL_TYPE, 0, NULL, 0,
				  WSA_FLAG_OVERLAPPED);
	
	if (listen_socket == INVALID_SOCKET){
		fprintf(stderr,"socket() failed with error %d\n",WSAGetLastError());
		return -1;
	}

	//
	// bind() associates a local address and port combination with the
	// socket just created. This is most useful when the application is a 
	// server that has a well-known port that clients know about in advance.
	//

	if (bind(listen_socket,(struct sockaddr*)&local,sizeof(local) ) 
		== SOCKET_ERROR) {
		fprintf(stderr,"bind() failed with error %d\n",WSAGetLastError());
		return -1;
	}

	msg_setopt(listen_socket);

	if (listen(listen_socket,5) == SOCKET_ERROR) {
	  fprintf(stderr,"listen() failed with error %d\n",WSAGetLastError());
	  return -1;
	}

	while(1) {
		char *name;
		struct hostent *p;

		cm_log(("Accepting connections\n"));

		fromlen =sizeof(from);
		msgsock = accept(listen_socket,(struct sockaddr*)&from, &fromlen);
		if (msgsock == INVALID_SOCKET) {
		  fprintf(stderr,"accept() error %d\n",WSAGetLastError());
		  return -1;
		}

		name = inet_ntoa(from.sin_addr);
		p = gethostbyaddr((char *)&from.sin_addr, 4, AF_INET);
		if (p == NULL) {
			printf("can't find host name %s %d\n", name, GetLastError());
			closesocket(msgsock);
			continue;
		}
		name = p->h_name;
		if (strchr(name, '~')) {
		    name = strchr(name, '~') + 1;
		}
		// find node id
		for (i = 0; i < NodesSize; i++) {
			int j;
			j = Strncasecmp(nodes[i], name, strlen(name));
			if (j == 0)
				break;
		}

		if (i < NodesSize) {
		    cm_log(("Accepted node : %d\n", i));

		    msg_setopt(msgsock);

		    tmp_socks[i] = msgsock;
		    CreateThread(NULL, 2*64*1024, srv_io,
				 (LPVOID) ((ULONGLONG)i), 0, NULL);
		} else {
			printf("bad node name: %d %s\n", i, name);
			closesocket(msgsock);
		}
	}
	return (0);
}

DWORD WINAPI
cmgr(LPVOID arg)
{
	int retval;
	struct sockaddr_in server;
	SOCKET  conn_socket;
	unsigned short port = (unsigned short) DEFAULT_PORT;
	int nodeid = (int) ((ULONGLONG)arg);
	char *server_name = nodes[nodeid];

	if (send_handles[nodeid] != 0 || (nodeid+1 == MY_NODEID))
	    return 0;

	memset(&server,0,sizeof(server));
	if (msg_buildaddr(&server, server_name, ipaddr[nodeid])) {
	    fprintf(stderr,"Client: cann't resolve name %s\n", server_name);
	    return 0;
	}

	//
	// Copy the resolved information into the sockaddr_in structure
	//
	server.sin_family = AF_INET; //hp->h_addrtype;
	server.sin_port = htons(port);

 again:	

	ResetEvent(Msg_Event[nodeid]);

	/* Open a socket */
	conn_socket = WSASocket(AF_INET, PROTOCOL_TYPE, 0, NULL, 0,
				WSA_FLAG_OVERLAPPED);
	if (conn_socket  != 0 ) {
		cm_log(("Client connecting to: %s\n", nodes[nodeid]));

		msg_setopt(conn_socket);

		if (connect(conn_socket,(struct sockaddr*)&server,sizeof(server))
		    != SOCKET_ERROR) {

		    cm_log(("Client connected to: %s\n", nodes[nodeid]));
		    GsLockEnter(msglock);
		    gs_node_handler[MSG_NODE_JOIN](nodeid+1);
		    send_handles[nodeid] = conn_socket;
		    if (!rcv_handles[nodeid]) {
			gs_node_handler[MSG_NODE_UP](nodeid+1);
		    }
		    GsLockExit(msglock);
		    srv_msg(conn_socket, nodeid);
		    msg_closenode(nodeid);
		    gs_node_handler[MSG_NODE_DOWN](nodeid+1);
		} else {
		    int err = WSAGetLastError();
		    cm_log(("connect() failed: %d\n", err));
		    closesocket(conn_socket);
		}
	} else {
	    int err = WSAGetLastError();
	    printf("Client: Error Opening socket: Error %d\n", err);
	}

	cm_log(("Cmgr %d sleeping\n", nodeid));
	WaitForSingleObject(Msg_Event[nodeid], INFINITE); //5 * 1000);
	cm_log(("Cmgr %d wokeup\n", nodeid));


	goto again;

    return (0);

}



static char *MCAST_IPADDR="224.0.20.65";
static int MPORT_NUM=9100;

int
OpenSocket(SOCKET *rs, struct sockaddr_in *sin, ULONG mipaddr, u_short port)
{
	struct hostent *h;
	int size, msgsize, len, n;
	struct sockaddr_in mysin;
	SOCKET s;
	char hostname[128];
	BOOLEAN bFlag = TRUE;

	s = WSASocket(AF_INET, SOCK_DGRAM, 0, (LPWSAPROTOCOL_INFO)NULL, 0, 
		  WSA_FLAG_OVERLAPPED | WSA_FLAG_MULTIPOINT_C_LEAF | WSA_FLAG_MULTIPOINT_D_LEAF);

	if (s == INVALID_SOCKET) {
	  fprintf(stderr, "Unable to create socket %d\n", GetLastError());
	  return 1;
	}
#if 1	
	if (msg_buildaddr(&mysin, nodes[MY_NODEID-1], ipaddr[MY_NODEID-1])) {
	    fprintf(stderr, "Unable to get my own address\n");
	    return 1;
	}
#if 0
	gethostname(hostname, sizeof(hostname));
	h = gethostbyname(hostname);
	if (h == NULL) {
		fprintf(stderr,"cannot get my own address\n");
		return 1;
	}
	printf("host %s addr cnt = %d\n", hostname, h->h_length);
	{
	    int i;
	    char *p;
	    p = (char *) h->h_addr_list[0];
	    for (i = 0; h->h_addr_list[i]; i++) {
		struct in_addr x;
		char *tmp;

		memcpy(&x, p, h->h_length);
		tmp = inet_ntoa(x);
		printf("Slot %d ip %s\n", i, tmp);
		p += h->h_length;
	    }
	}
	memcpy(&mysin.sin_addr.s_addr, h->h_addr_list[0], 4);
#endif
#else
	if (ipaddr[MY_NODEID-1] != NULL) {
	    mysin.sin_addr.s_addr = inet_addr(ipaddr[MY_NODEID-1]);
	} else {
	    mysin.sin_addr.s_addr = INADDR_ANY;
	}
#endif
	mysin.sin_family = PF_INET;
	port = htons (port);
	mysin.sin_port = (u_short) port;

	if (bind (s, (struct sockaddr *)&mysin, sizeof(mysin)) <0) {
	  fprintf(stderr, "Bind failed %d\n", GetLastError());
	  return 1;
	}

	len = sizeof(max_mcmsg);
	/* get max. message size */
	if (getsockopt(s, SOL_SOCKET, SO_MAX_MSG_SIZE, (PVOID) &max_mcmsg,
		       &len)) {
	    fprintf(stderr,"getsockopt SO_MAX_MSG_SIZE failed %d\n",
		    WSAGetLastError());
	    closesocket(s);
	    return 1;
	}
	max_mcmsg -= sizeof(gs_msg_hdr_t);
	printf("Max mcast message %d\n", max_mcmsg);

	/* make sure we can run multiple copies */
	if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char *) &bFlag, sizeof(bFlag))< 0) {
	  fprintf(stderr, "setsockopt SO_REUSEADDR failed %d\n", GetLastError());
	  closesocket(s);
	  return 1;
	}

	/* disable loopback on send */
	bFlag = FALSE;
	if (WSAIoctl(s, SIO_MULTIPOINT_LOOPBACK, (char *) &bFlag, sizeof(bFlag), NULL, 0, &n, NULL, NULL)< 0) {
	  fprintf(stderr, "ioctl loopback failed %d\n", GetLastError());
	  closesocket(s);
	  return 1;
	}

	sin->sin_family      = PF_INET;
	sin->sin_port        = (u_short) (ntohs(port));
	sin->sin_addr.s_addr = mipaddr; //inet_addr(MCAST_IPADDR);

	/* join the multicast address */
	s = WSAJoinLeaf (s, (struct sockaddr *)sin, sizeof (*sin),
		NULL, NULL, NULL, NULL,  JL_BOTH); 

	/* dead in the water */
	if (s == INVALID_SOCKET) {
		fprintf(stderr, "Join failed %d\n", GetLastError());
		return 1;
	}

	*rs = s;
	return 0;
}


SOCKET msock;
struct sockaddr_in msin;

void
mcast_send(gs_msg_hdr_t *hdr, const char *buf, int len)
{
	int i;
	WSABUF	io[2];

	msg_log(("Send msg mcast type %d seq %d len %d\n",
		 hdr->h_type, hdr->h_mseq, len));

	io[0].buf = (char *) hdr;
	io[0].len = sizeof(*hdr);
	io[1].buf = (char *) buf;
	io[1].len = len;

	if (WSASendTo(msock, io, 2, &i, 0, 
		    (struct sockaddr *)&msin, sizeof(msin), 0, 0)) {
	    int err = WSAGetLastError();
	    if (err == WSA_IO_PENDING) {
		printf("Async send\n");
		return;
	    }
	    printf("Send failed %d\n", WSAGetLastError());
	    halt(1);
	}
	i -= sizeof(*hdr);
	if (i != len) {
	    printf("Send failed: mcast len %d, %d\n", len, i);
	    halt(1);
	}
	msg_log(("Send done mcast type %d seq %d\n",
		 hdr->h_type, hdr->h_mseq));
	return;
}

void
mcast_init()
{
  u_short port = (u_short) MPORT_NUM;
  ULONG	ipaddr = inet_addr(MCAST_IPADDR);

  if (OpenSocket(&msock, &msin, ipaddr, port) == 1) {
      err_log(("Unable to create mcast socket\n"));
      mcast_enabled = 0;
      max_mcmsg = 0;
  }

  printf("Mcast %d\n", mcast_enabled);



}

DWORD WINAPI
mcast_srv(LPVOID arg)
{
  SOCKET msgsock;
  gs_msg_t *msg;
  int retval;
  char *buf;
  int len, flags;

  msgsock = msock;
  while (1) {
      extern gs_msg_handler_t gs_msg_handler[];
      int type;
      WSABUF	io[2];

      msg = msg_alloc(NULL, GS_MAX_MSG_SZ);
      assert(msg);
      assert(msg->m_buflen != 0);

      io[0].buf = (char *)&msg->m_hdr;
      io[0].len = sizeof(msg->m_hdr);
      io[1].buf = msg->m_buf;
      io[1].len = msg->m_buflen;

      flags = 0;
      retval = WSARecv(msgsock, io, 2, &len, &flags, 0, 0);
      if (retval == SOCKET_ERROR) {
	  err_log(("mcast recv failed %d, %d, len %d\n",
		   retval,
		   WSAGetLastError(), msg->m_buflen));
	  msg_free(msg);
	  halt(1);
	  return 0;
      }

      if (len != (int)(msg->m_hdr.h_len + sizeof(msg->m_hdr))) {
	  err_log(("Bad mcast recv got %d, expected %d\n", len, msg->m_hdr.h_len));
	  halt(1);
      }
      msg_log(("rec mcast nid %d gid %d type %d seq %d view %d len %d\n",
	       msg->m_hdr.h_sid,msg->m_hdr.h_gid, type = msg->m_hdr.h_type,
	       msg->m_hdr.h_mseq, msg->m_hdr.h_viewnum, msg->m_hdr.h_len));

      gs_msg_handler[msg->m_hdr.h_type](msg);

      msg_log(("Done Type %d\n", type));
  }
}

void
msg_set_uport(int uport)
{
    DEFAULT_PORT = uport;
}

void
msg_set_mport(int mport)
{
    MPORT_NUM = mport;
}

void
msg_set_subnet(char *addr)
{
    strcpy(cl_subnet, addr);
}

void
msg_set_mipaddr(char *addr)
{
}

void
msg_set_bufcount(int count)
{
    MSG_POOL_SZ = count;
}

void
msg_set_bufsize(int size)
{
    if (size > GS_MAX_MSG_SZ) {
	fprintf(stderr,"You are exceeding the 64K msg size limit\n");
    } else {
	GS_MAX_MSG_SZ = size;
    }
}

void
msg_set_mode(int mode)
{
    mcast_enabled = mode;
}