Captain’s Mistress- An implementation of a simple game

Description:

The game “Captain’s Mistress”, also known as “Connect Four”.

https://en.wikipedia.org/wiki/Connect_Four

 

Standard Connect Four configuration and rules apply:

*   There are two players.

*   The board should be seven columns wide.

*   The board should be six rows high.

*   It takes four pieces in a row to win.

Your implementation of the game should meet the following criteria:

*   Players place their discs by entering a column number on the command line.

*   The board should be rendered after each turn.

*   If a winning move is played, the winning board should be rendered and the

winner should be displayed.

 

Ruby implementation:

 


require 'captains_mistress/version'

# A simple game of Captain's Mistress.
module CaptainsMistress
	  class Player
		@disc
		@steps
		@win
		
		def initialize(name)
			@name = name
			@steps = Array.new()
			@win = false
			
		end
		attr_reader :name
		attr_reader :disc
		attr_reader :steps
		attr_reader :win
		
	
		def setDisc(number)
			@disc = number
		end
	
		def setWin(win)
			@win = win
		end
	end

	class Engine
		@board = Array.new(6){Array.new(7,0)}
		@player1
		@player2
		@winner
		@gameOver
		@validColumnNumber
		
		def initialize(board, player1, player2)
			@board = board
			@player1 = player1
			@player2 = player2
			@validColumnNumber = ["0", "1", "2", "3", "4", "5", "6"]
			initialPlayerDiscs()
			decideFirst()
			renderBoard()
		end
		attr_reader :board
		attr_reader :player1
		attr_reader :player2
		attr_reader :winner
		attr_reader :gameOver
		attr_reader :validColumnNumber
	
		def initialPlayerDiscs()
			random = rand(1..100)
			if(random%2 == 0)
				@player1.setDisc(1) 
				@player2.setDisc(2)
			else
				@player1.setDisc(2) 
				@player2.setDisc(1)
			end		
			getPlayerDisc()
		end
	
		#decide who goes first
		def decideFirst()
			random = rand(1..100)
			if(random%2 == 0) 
				firstHand = @player1
			else
				firstHand = @player2
			end
			puts "#{firstHand::name} goes first"
			puts ""		
		end
	
		def addDisc(player, columnIndex)
		puts "columnIndex = #{columnIndex}"
		if(!@validColumnNumber.include?(columnIndex))
			warnningOfInvalid(player)
			return false;
		else
			columnIndex = columnIndex.to_i
		end
		rowIndex = updateBoard(player, columnIndex)
			if(columnIndex >= @board[0].length || columnIndex <0)
				warnningOfInvalid(player)
				return false
			elsif(rowIndex == -1)
				warnningOfInvalid(player)
				return false
			elsif(checkWin(player, columnIndex, rowIndex))
				setWinner(player) 
				@gameOver = true
			else
			end
			updatePlayerStatus(player, columnIndex)
			if(@gameOver) 
				renderFinalResult()
				exit(1)
			else
				renderBoard()
			end
				puts "addDisc successfully"
				puts ""
			return true	
		end
	
	
		def updateBoard(player, columnIndex)
			rowIndex = @board.length
			if(@board[0][columnIndex] != 0) 
				warnningOfInvalid(player)
				return -1
			else
				@board.each_index do |i|
					if @board[@board.length-i-1][columnIndex] == 0
						@board[@board.length-i-1][columnIndex] = player.disc
						rowIndex = @board.length-i-1
						break
					else
					end	
				end
			end	
			return rowIndex
		end
	
	
		def warnningOfInvalid(player)
			puts "Hey #{player::name},  your input is invalid"
		end	
	
		def updatePlayerStatus(player, columnIndex)
			player::steps.push(columnIndex)
			if(@winner ==  player) 
				player.setWin(true)
			else
			end
		end
	
	
		def setWinner(player) 
			@winner = player;
		end
	
		def checkWin(player, columnIndex, rowIndex)
			if(checkColumn(player, columnIndex, rowIndex) || checkRow(player, columnIndex, rowIndex) || checkLeftDiagonal(player, columnIndex, rowIndex) || checkRightDiagonal(player, columnIndex, rowIndex))
				return true
			else
			end
			return false;
		end
		
		def checkColumn(player, columnIndex, rowIndex) 
			disc = player::disc
			i = rowIndex-1
			count = 1
				while(i >= 0 && @board[i][columnIndex] == disc) 
					i -= 1
					count +=1
					if(count == 4) 
						return true
					else
					end
				end
				i = rowIndex+1
				while(i < @board.length && @board[i][columnIndex] == disc) 
					i += 1
					count += 1
					if(count == 4) 
					return true
					else
					end
				end
			return false
		end
	
		def checkRow(player, columnIndex, rowIndex) 
			disc = player::disc
			i = columnIndex-1
			count = 1;
				while(i >= 0 && @board[rowIndex][i] == disc) 
					i -=1
					count +=1
					if(count == 4) 
					return true
					else
					end
				end
				i = columnIndex+1
				while(i < @board[rowIndex].length && @board[rowIndex][i] == disc) 
					i += 1
					count += 1
					if(count == 4)
					return true
					else
					end
				end		
			return false
		end
	
		def checkLeftDiagonal(player, columnIndex, rowIndex) 
			disc = player::disc
			i = rowIndex-1
			j = columnIndex-1
			count = 1
				while(i >= 0 && j >= 0 && @board[i][j] == disc) 
					i -=1
					j -= 1
					count +=1
					if(count == 4) 
					return true
					else
					end
				end
				i = rowIndex+1
				j = columnIndex+1
				while(i < @board.length && j < @board[0].length && @board[i][j] == disc) 
					i += 1
					j +=1
					count += 1
					if(count == 4) 
						return true
					else
					end
				end			
			return false;
		end
	
		def checkRightDiagonal(player, columnIndex, rowIndex) 
			disc = player::disc
			i = rowIndex-1
			j = columnIndex+1
			count = 1
				while(i >= 0 && j < @board.length && @board[i][j] == disc) 
					i -=1
					j +=1
					count +=1
					if(count == 4) 
					return true
					else
					end
				end
				i = rowIndex=1
				j = columnIndex-1
				while(i < @board.length && j >= 0 && @board[i][j] == disc) 
					i += 1
					j -=1
					count += 1
					if(count == 4) 
						return true
					else
					end
				end			
			return false
		end
	
	
		def renderBoard()
		#index for each column
			puts "0 1 2 3 4 5 6"
			puts "V V V V V V V "
			
			@board.each do |r|
				puts r.each { |p| p }.join(" ")
			end
			puts ""
		end 
	
		def renderFinalResult()
			puts "" 
			puts "Game Over!!!!"
			renderBoard()
			
			puts "The winner is: #{@winner::name}. Congratulations!!!!!"
			puts ""
		end
	
		def getPlayerDisc()
			puts "#{@player1::name}'s disc is #{@player1::disc}" 	
			puts "#{@player2::name}'s disc is #{@player2::disc}" 			
		end
	
		def gameOver()
			return @gameOver
		end
	
		def getWinner()
			return @winner
		end
	end
end


require 'captains_mistress'

module CaptainsMistress
  # The application object. Create it with options for the game, then run by
  # calling #run.
  class App
    attr_reader :verbose

    def initialize(options = {})
      @verbose = options.fetch(:verbose, false)
    end

    def run
		puts "Game of 'Captain's Mistress' "
		puts ""
		puts "Player1, please enter your name: "
		p1Name = $stdin.gets.chomp
		puts ""
		player1 = Player.new(p1Name)
		puts "Player2, please enter your name: "
		p2Name = $stdin.gets.chomp
		player2 = Player.new(p2Name)
		puts ""
		board= Array.new(6){Array.new(7,0)}
		game1 = Engine.new(board, player1, player2)

		if(game1.decideFirst() == player2.name) 
			firstHand = plasyer2;
			secondHand = player1;
		else
			firstHand = player1;
			secondHand = player2;
		end

		round = 0

		while(!game1.gameOver())
			round += 1
			puts "Round #{round}"
			firstColumnIndex = -1
				begin		
				puts "#{firstHand::name}, enter your next step: (select a column number from 0 to 6)"
				firstColumnIndex = $stdin.gets.chomp
				end while (!game1.gameOver() && !game1.addDisc(firstHand, firstColumnIndex))
			if(game1.gameOver()) 
				break
			else
			end
			secondColumnIndex = -1
			begin
				puts "#{secondHand::name}, enter your next step: (select a column number from 0 to 6)"
				secondColumnIndex = $stdin.gets.chomp
				end while (!game1.gameOver() && !game1.addDisc(secondHand, secondColumnIndex))
				puts ""
		end
    end
  end
end

 

Java implementation:

 

import java.util.List;

/*A class that represents a player.
 *Include's an integer 0 or 1 to present a player's disc type;
 *A boolean to indicate a player is win or not;
 *A list of integers to presents each step a player goes in a game. The integers are the column indexes.*/


public class Player {
	String name;
	int disc;
	boolean win;
	List<Integer> steps;
	public Player(String name) {
		this.name = name;
	}
}


import java.util.ArrayList;
import java.util.Random;


public class Game_Implementation {
	private int[][] board;
	private Player player1, player2, winner;
	private boolean gameOver;

	// initialization and validation of the game
	public Game_Implementation(int[][] board, Player player1, Player player2) {
		if(board == null || board.length != 6 || board[0].length != 7) {
			System.out.println("This is not a valid baord. The game board should have 6 rows and 7 columns");
		}else{
			this.board = board;
			this.player1 = player1;
			this.player2 = player2;
			initialPlayer();
		}		
	}
	
	private void initialPlayer() {
		player1.steps = new ArrayList<>();
		player2.steps = new ArrayList<>();
		initialPlayerDiscs();
	}
	
	private void initialPlayerDiscs(){
		int random =  (int) Math.ceil(Math.random() * 100) ;
		if(random%2 == 0) {
			player1.disc = 1;
			player2.disc = 2;
		}else {
			player1.disc = 2;
			player2.disc = 1;
		}
		System.out.println(player1.name +"'disc is "+player1.disc);
		System.out.println(player2.name +"'disc is "+player2.disc);
	}
	// decide who plays first.
	public String decideFirst() {
		Random firstLot = new Random();
		int random = firstLot.nextInt(100);
		if(random%2 == 1) return player1.name;
		return player2.name;
	}
	
	public boolean addDisc(Player player, int columnIndex) {
		if(columnIndex >= board[0].length || columnIndex <0) {
			warnningOfInvalid(player);
			return false;
		}
		int rowIndex = updateBoard(player, columnIndex);
		if(rowIndex == -1) return false;
		if(checkWin(player, columnIndex, rowIndex)){
			setWinner(player);
			gameOver = true;
		}
		updatePlayerStatus(player, columnIndex);
		if(gameOver) {
			renderFinalResult();
		}else{
			renderBoard();
		}
		return true;
	}
	
	private int updateBoard(Player player, int columnIndex) {
		int rowIndex = board.length;
		if(board[0][columnIndex] != 0) {
			warnningOfInvalid(player);
			return -1;
		}else{
			for(int i = board.length-1; i >= 0; i--) {
				if(board[i][columnIndex] == 0){
					board[i][columnIndex] = player.disc;
					rowIndex = i;
					break;
				}		
			}
		}	
		return rowIndex;
	}
	
	private void warnningOfInvalid (Player player) {
		System.out.println("Hey "+ player.name+", your movement is invalid");
	}
	private void updatePlayerStatus(Player player, int columnIndex) {
		player.steps.add(columnIndex);
		if(winner ==  player) player.win = true;
	}
	
	
	private void setWinner(Player player) {
		winner = player;
	}
	
	private boolean checkWin(Player player, int columnIndex, int rowIndex) {
		if(checkColumn(player, columnIndex, rowIndex) 
				|| checkRow(player, columnIndex, rowIndex) 
				|| checkLeftDiagonal(player, columnIndex, rowIndex) 
				|| checkRightDiagonal(player, columnIndex, rowIndex))
			return true;
		return false;
	}
	
	private boolean checkColumn(Player player, int columnIndex, int rowIndex) {
		int disc = player.disc;
		int i = rowIndex;
		int count = 1;
			while(--i >= 0 && board[i][columnIndex] == disc) {
				++count;
				if(count == 4) return true;
			}
			i = rowIndex;
			while(++i < board.length && board[i][columnIndex] == disc) {
				++count;
				if(count == 4) return true;
			}
		return false;
	}
	
	private boolean checkRow(Player player, int columnIndex, int rowIndex) {
		int disc = player.disc;
		int i = columnIndex;
		int count = 1;
			while(--i >= 0 && board[rowIndex][i] == disc) {
				++count;
				if(count == 4) return true;
			}
			i = columnIndex;
			while(++i < board[rowIndex].length && board[rowIndex][i] == disc) {
				++count;
				if(count == 4) return true;
			}
		return false;
	}
	
	private boolean checkLeftDiagonal(Player player, int columnIndex, int rowIndex) {
		int disc = player.disc;
		int i = rowIndex, j = columnIndex;
		int count = 1;
			while(--i >= 0 && --j >= 0 && board[i][j] == disc) {
				++count;
				if(count == 4) return true;
			}
			i = rowIndex;
			j = columnIndex;
			while(++i < board.length && ++j < board[0].length && board[i][j] == disc) {
				++count;
				if(count == 4) return true;
			}			
		return false;
	}
	
	private boolean checkRightDiagonal(Player player, int columnIndex, int rowIndex) {
		int disc = player.disc;
		int i = rowIndex, j = columnIndex;
		int count = 1;
			while(--i >= 0 && ++j < board.length && board[i][j] == disc) {
				++count;
				if(count == 4) return true;
			}
			i = rowIndex;
			j = columnIndex;
			while(++i < board.length && --j >= 0 && board[i][j] == disc) {
				++count;
				if(count == 4) return true;
			}			
		return false;
	}
	
	
	public void renderBoard(){
		for( int i = 0; i < board.length; i++) {
			for(int j = 0; j < board[0].length; j++) {
				System.out.print(board[i][j]+ " ");
			}
			System.out.println();
		}
		System.out.println();
	}
	
	public void renderFinalResult() {
		System.out.println("Game Over");
		renderBoard();
		System.out.println("The winner is: "+ winner.name);
	}
	
	public void getPlayerDisc() {		
		System.out.println(player1.name +"'s disc is "+ player1.disc);
		System.out.println(player2.name +"'s disc is "+ player2.disc);
	}
	
	public boolean gameOver(){
		return gameOver;
	}
	
	public Player getWinner(){
		return winner;
	}
	
}



import java.util.Scanner;

public class PlayTheGame {
	public static void main(String args[]) {
		
		System.out.println("Game of 'Captain's Mistress' ");
		@SuppressWarnings("resource")
		Scanner scanner = new Scanner(System.in);
		int[][] board = new int[6][7];
		System.out.println("Player1, please enter your name: ");
		String p1Name = scanner.nextLine();
		Player player1 = new Player(p1Name);
		System.out.println("Player2, please enter your name: ");
		String p2Name = scanner.nextLine();
		Player player2 = new Player(p2Name);
		System.out.println();

		Game_Implementation game1 = new Game_Implementation(board,player1, player2);
		System.out.println();

		Player firstHand = player1;
		Player secondHand = player2;
		if(game1.decideFirst() == player2.name) {
			firstHand = player2;
			secondHand = player1;
		}
		
		System.out.println();
		System.out.println(firstHand.name + " gose first");
		System.out.println();
		
		int round = 0;
		while(!game1.gameOver()) {
			System.out.println("TestRound = "+(round++));
			
			int firstColumnIndex;
			do{
				System.out.println( firstHand.name + ", enter your next step: ");
				firstColumnIndex = scanner.nextInt();
				System.out.println();

			}
			while(!game1.gameOver() && !game1.addDisc(firstHand, firstColumnIndex));			
			if(game1.gameOver()) break;
			int secondColumnIndex;
			do{
				System.out.println(secondHand.name + ", enter your next step: ");
				secondColumnIndex = scanner.nextInt();
				System.out.println();

			}
			while(!game1.gameOver() && !game1.addDisc(secondHand, secondColumnIndex));
		}
		
		System.out.println("Congratulations to "+ game1.getWinner().name);
	}

}

318. Maximum Product of Word Lengths

https://leetcode.com/problems/maximum-product-of-word-lengths/

 

Given a string array words, find the maximum value of length(word[i]) * length(word[j]) where the two words do not share common letters. You may assume that each word will contain only lower case letters. If no such two words exist, return 0.

Example 1:

Given ["abcw", "baz", "foo", "bar", "xtfn", "abcdef"]
Return 16
The two words can be "abcw", "xtfn".

Example 2:

Given ["a", "ab", "abc", "d", "cd", "bcd", "abcd"]
Return 4
The two words can be "ab", "cd".

Example 3:

Given ["a", "aa", "aaa", "aaaa"]
Return 0
No such pair of words.

 

Solution:

The intuition for solving this problem is to compare each pair of words. The complexity of iterating each pair of words is O(n^2). If we compare each character in two words, there will be an additional O(k^2) complexity. (n is the number of words, k is the  length of a word).

Thus, The brutal force solution is time consuming. So we need to find out a way to decrease the complexity of previous two steps. The comparison between each two words is hard to optimize, since the words are arbitrary. However, there is a trick we can do on the comparison of characters in each word.

The idea is use an integer’s lower 26 bits to present a character of a word. if a character exists in a word, the bit that represent that word will be 1, otherwise 0. Thus, after this pre-processing, when we and any two integers, if there do not have same characters, the result will be 0, otherwise will be 1.

Hence, the complexity of character’s comparison will be O(1), which make the whole time complexity be O(n^2). This is acceptable.

Here is the code:


/*
bit manipulation.

*/
public class Solution {
    public int maxProduct(String[] words) {
        int len = words.length;
        int max = 0;
        if(len == 0) return 0;
        int[] biVals = new int[len];
  // pre-processing each word, use an integer's lower 26 bits to present a word's characters.
        for(int i = 0; i < len; i++) {
            biVals[i] = 0;
            for(char ch : words[i].toCharArray()) {
                biVals[i] |= 1 << (ch - 'a');
            }
        }
    // compare each pair of words in O(n^2).
        for( int i = 0; i < len; i++) {
            for(int j = i+1; j < len; j++) {                 if(((biVals[i] & biVals[j]) == 0) && (words[i].length() * words[j].length() > max))
                    max = words[i].length() * words[j].length();
            }
        }
        return max;
    }
}

 

 

 

Leetcode 284. Peeking Iterator

https://leetcode.com/problems/peeking-iterator/

 

Given an Iterator class interface with methods: next() and hasNext(), design and implement a PeekingIterator that support the peek() operation — it essentially peek() at the element that will be returned by the next call to next().


Here is an example. Assume that the iterator is initialized to the beginning of the list: [1, 2, 3].

Call next() gets you 1, the first element in the list.

Now you call peek() and it returns 2, the next element. Calling next() after that still return 2.

You call next() the final time and it returns 3, the last element. Calling hasNext() after that should return false.

 

Solutions:

Here I come up with two solutions. One is according to Google’s guava library source code.

The other one is using List, which may be slower, because we need to load all element in the iterator when we construct the class.

solution1:

class PeekingIterator implements Iterator<Integer> {
    Iterator it;
    int peek;
    boolean peeked;
	public PeekingIterator(Iterator<Integer> iterator) {
	    // initialize any member here.
	    it = iterator;
	}

    // Returns the next element in the iteration without advancing the iterator.
	public Integer peek() {
	    if(!peeked) {
	        peek = (int)it.next();
	        peeked = true;
	    }

        return peek;
	}

	// hasNext() and next() should behave the same as in the Iterator interface.
	// Override them if needed.
	@Override
	public Integer next() {
	    if(!peeked) return (int)it.next();
	    int result = peek;
	    peeked = false;
	    return result;
	}

	@Override
	public boolean hasNext() {
	    return peeked || it.hasNext();
	}
}

 

solution2:

// Java Iterator interface reference:
// https://docs.oracle.com/javase/8/docs/api/java/util/Iterator.html
class PeekingIterator implements Iterator<Integer> {
    List<Integer> iteratorList = new ArrayList<>();
    int pointer;
	public PeekingIterator(Iterator<Integer> iterator) {
	    // initialize any member here.
	    pointer = 0;
	    while(iterator.hasNext()) {
	        iteratorList.add(iterator.next());
	    }
	}

    // Returns the next element in the iteration without advancing the iterator.
	public Integer peek() {
        return iteratorList.get(pointer);
	}

	// hasNext() and next() should behave the same as in the Iterator interface.
	// Override them if needed.
	@Override
	public Integer next() {
	    return iteratorList.get(pointer++);
	}

	@Override
	public boolean hasNext() {
	    return pointer < iteratorList.size();
	}
}

leetcode 288. Unique Word Abbreviation

https://leetcode.com/problems/unique-word-abbreviation/

 

An abbreviation of a word follows the form <first letter><number><last letter>. Below are some examples of word abbreviations:

a) it                      --> it    (no abbreviation)

     1
b) d|o|g                   --> d1g

              1    1  1
     1---5----0----5--8
c) i|nternationalizatio|n  --> i18n

              1
     1---5----0
d) l|ocalizatio|n          --> l10n

Assume you have a dictionary and given a word, find whether its abbreviation is unique in the dictionary. A word’s abbreviation is unique if no other word from the dictionary has the same abbreviation.

Example:

Given dictionary = [ "deer", "door", "cake", "card" ]

isUnique("dear") -> false
isUnique("cart") -> true
isUnique("cane") -> false
isUnique("make") -> true

Solution:

First solution is that use a set of String for an abbr. HashMap<String, Set<String>> abbrDictionary;

The improvement is that when a duplicate happens, use a null to present the value of a abbr. HashMap<String, String> abbrDictionary;

public class ValidWordAbbr {
    HashMap<String, Set<String>> abbrDictionary;
    public ValidWordAbbr(String[] dictionary) {
        abbrDictionary = new HashMap<>();
        for(String word : dictionary) {
            String key = getKey(word);
            if(!abbrDictionary.containsKey(key)) abbrDictionary.put(key, new HashSet<String>());
            abbrDictionary.get(key).add(word);
        }
    }

    public boolean isUnique(String word) {
        String key = getKey(word);
        return !abbrDictionary.containsKey(key)|| (abbrDictionary.get(key).size() ==1 && abbrDictionary.get(key).contains(word));
    }

     private String getKey(String str){
        if(str.length()<=2) return str;
        return str.charAt(0)+Integer.toString(str.length()-2)+str.charAt(str.length()-1);
    }

}

// Your ValidWordAbbr object will be instantiated and called as such:
// ValidWordAbbr vwa = new ValidWordAbbr(dictionary);
// vwa.isUnique("Word");
// vwa.isUnique("anotherWord");
public class ValidWordAbbr {
    HashMap<String, String> map;
    public ValidWordAbbr(String[] dictionary) {
        map = new HashMap<String, String>();
        for(String str:dictionary){
            String key = getKey(str);
            // If there is more than one string belong to the same key
            // then the key will be invalid, we set the value to ""
            if(map.containsKey(key)){
                if(!map.get(key).equals(str)){
                    map.put(key, "");
                }
            }
            else{
                map.put(key, str);
            }
        }
    }

    public boolean isUnique(String word) {
        return !map.containsKey(getKey(word))||map.get(getKey(word)).equals(word);
    }

    String getKey(String str){
        if(str.length()<=2) return str;
        return str.charAt(0)+Integer.toString(str.length()-2)+str.charAt(str.length()-1);
    }
}

// Your ValidWordAbbr object will be instantiated and called as such:
// ValidWordAbbr vwa = new ValidWordAbbr(dictionary);
// vwa.isUnique("Word");
// vwa.isUnique("anotherWord");

Hadoop

When we talk about distributed systems, Hadoop maybe the most popular architecture nowadays, although it is not the first distributed system.

What Is Apache Hadoop?

cited from: http://hadoop.apache.org

The Apache™ Hadoop® project develops open-source software for reliable, scalable, distributed computing.

The Apache Hadoop software library is a framework that allows for the distributed processing of large data sets across clusters of computers using simple programming models. It is designed to scale up from single servers to thousands of machines, each offering local computation and storage. Rather than rely on hardware to deliver high-availability, the library itself is designed to detect and handle failures at the application layer, so delivering a highly-available service on top of a cluster of computers, each of which may be prone to failures.

 

Hadoop includes four modules:

  • Hadoop Common: The common utilities that support the other Hadoop modules.
  • Hadoop Distributed File System (HDFS™): A distributed file system that provides high-throughput access to application data.
  • Hadoop YARN: A framework for job scheduling and cluster resource management.
  • Hadoop MapReduce: A YARN-based system for parallel processing of large data sets.

Following is the introduction and architecture for each part.

Hadoop Common:

These are Java libraries and utilities required by other Hadoop modules. These libraries provides filesystem and OS level abstractions and contains the necessary Java files and scripts required to start Hadoop.

Hadoop Distributed File System (HDFS):

hdfsarchitecture

 

HDFS follows the master-slave architecture and it has the following elements.

Namenode

The namenode is the commodity hardware that contains the GNU/Linux operating system and the namenode software. It is a software that can be run on commodity hardware. The system having the namenode acts as the master server and it does the following tasks:

  • Manages the file system namespace.
  • Regulates client’s access to files.
  • It also executes file system operations such as renaming, closing, and opening files and directories.

Datanode

The datanode is a commodity hardware having the GNU/Linux operating system and datanode software. For every node (Commodity hardware/System) in a cluster, there will be a datanode. These nodes manage the data storage of their system.

  • Datanodes perform read-write operations on the file systems, as per client request.
  • They also perform operations such as block creation, deletion, and replication according to the instructions of the namenode.

Block

Generally the user data is stored in the files of HDFS. The file in a file system will be divided into one or more segments and/or stored in individual data nodes. These file segments are called as blocks. In other words, the minimum amount of data that HDFS can read or write is called a Block. The default block size is 64MB, but it can be increased as per the need to change in HDFS configuration.

Hadoop YARN

cited from: http://ercoppa.github.io/HadoopInternals/HadoopArchitectureOverview.html

yarn_architecture

the YARN Infrastructure (Yet Another Resource Negotiator) is the framework responsible for providing the computational resources (e.g., CPUs, memory, etc.) needed for application executions. Two important elements are:

  • the Resource Manager (one per cluster) is the master. It knows where the slaves are located (Rack Awareness) and how many resources they have. It runs several services, the most important is the Resource Scheduler which decides how to assign the resources.Services provided by Resource Manager is showed below:

resource-manager_534be06c-eb4c-4516-a178-5ff00a005d90

 

  • the Node Manager (many per cluster) is the slave of the infrastructure. When it starts, it announces himself to the Resource Manager. Periodically, it sends an heartbeat to the Resource Manager. Each Node Manager offers some resources to the cluster. Its resource capacity is the amount of memory and the number of vcores. At run-time, the Resource Scheduler will decide how to use this capacity: a Container is a fraction of the NM capacity and it is used by the client for running a program.

node-manager-overview_534beb08-0c0c-4d84-bf75-3a670a00c014

In YARN, there are at least three actors:

  • the Job Submitter (the client)
  • the Resource Manager (the master)
  • the Node Manager (the slave)

The application startup process is the following:

  1. a client submits an application to the Resource Manager
  2. the Resource Manager allocates a container
  3. the Resource Manager contacts the related Node Manager
  4. the Node Manager launches the container
  5. the Container executes the Application Master

The Application Master is responsible for the execution of a single application. It asks for containers to the Resource Scheduler (Resource Manager) and executes specific programs (e.g., the main of a Java class) on the obtained containers. The Application Master knows the application logic and thus it is framework-specific. The MapReduce framework provides its own implementation of an Application Master.

The Resource Manager is a single point of failure in YARN. Using Application Masters, YARN is spreading over the cluster the metadata related to running applications. This reduces the load of the Resource Manager and makes it fast recoverable.

yarn-architecture_5356ab97-2bd8-4f19-b30e-1ef60a00dcc0yarn-application-startup_534bf195-890c-4c7a-95eb-13cb0a008d03

 

Hadoop MapReduce

cited from: http://www.tutorialspoint.com/hadoop/hadoop_mapreduce.htm; https://en.wikipedia.org/wiki/MapReduce

What is MapReduce?

MapReduce is a processing technique and a program model for distributed computing based on java. The MapReduce algorithm contains two important tasks, namely Map and Reduce. Map takes a set of data and converts it into another set of data, where individual elements are broken down into tuples (key/value pairs). Secondly, reduce task, which takes the output from a map as an input and combines those data tuples into a smaller set of tuples. As the sequence of the name MapReduce implies, the reduce task is always performed after the map job.

Why MapReduce?

The major advantage of MapReduce is that it is easy to scale data processing over multiple computing nodes. Under the MapReduce model, the data processing primitives are called mappers and reducers. Decomposing a data processing application into mappers and reducers is sometimes nontrivial. But, once we write an application in the MapReduce form, scaling the application to run over hundreds, thousands, or even tens of thousands of machines in a cluster is merely a configuration change. This simple scalability is what has attracted many programmers to use the MapReduce model.

What the steps of MapReduce?

  • “Map” step: Each worker node applies the “map()” function to the local data, and writes the output to a temporary storage. A master node ensures that only one copy of redundant input data is processed.
  • “Shuffle” step: Worker nodes redistribute data based on the output keys (produced by the “map()” function), such that all data belonging to one key is located on the same worker node.
  • “Reduce” step: Worker nodes now process each group of output data, per key, in parallel.

Another way to look at MapReduce is as a 5-step parallel and distributed computation:

  1. Prepare the Map() input – the “MapReduce system” designates Map processors, assigns the input key value K1 that each processor would work on, and provides that processor with all the input data associated with that key value.
  2. Run the user-provided Map() code – Map() is run exactly once for each K1 key value, generating output organized by key values K2.
  3. “Shuffle” the Map output to the Reduce processors – the MapReduce system designates Reduce processors, assigns the K2 key value each processor should work on, and provides that processor with all the Map-generated data associated with that key value.
  4. Run the user-provided Reduce() code – Reduce() is run exactly once for each K2 key value produced by the Map step.
  5. Produce the final output – the MapReduce system collects all the Reduce output, and sorts it by K2 to produce the final outcome.

These five steps can be logically thought of as running in sequence – each step starts only after the previous step is completed – although in practice they can be interleaved as long as the final result is not affected.

hackerrank: Breadth First Search: Shortest Reach

Problem:

https://www.hackerrank.com/challenges/bfsshortreach

Analysis:

Just a implementation of bfs.

 

Solution:

 


import java.io.*;
import java.util.*;
import java.text.*;
import java.math.*;
import java.util.regex.*;

public class Solution {
    static List[] graph;

    @SuppressWarnings("unchecked")
	public static void addToGraph(int node1, int node2) {
        if(graph[node1] == null) graph[node1] = new ArrayList<Integer>();
        if(!graph[node1].contains(node2))
        	graph[node1].add(node2);
        if(graph[node2] == null) graph[node2] = new ArrayList<Integer>();
        if(!graph[node2].contains(node1))
        graph[node2].add(node1);
    }    

    public static void main(String[] args) {
        /* Enter your code here. Read input from STDIN. Print output to STDOUT. Your class should be named Solution. */
        Scanner in = new Scanner(System.in);
        int num_cases = in.nextInt();
        for(int c = 0; c < num_cases; c++) {
    	int num_nodes = in.nextInt(), num_edge = in.nextInt();
        graph = new List[num_nodes+1];
    	for(int i = 0; i < num_edge; i++) {
            addToGraph(in.nextInt(), in.nextInt());
        }
    		List<Integer> result = new ArrayList<>();
            int startNode = in.nextInt(); //get the start node.

            // print the graph for test
            /*
            System.out.println("This is the graph represented using adjacent lists");
            for(int x = 0; x < graph.length; x++) {                 System.out.print(x + "--> ");
                System.out.println(graph[x]);
            }*/

            // get the shortest distance for each destinationNode.
            for(int destinationNode = 1; destinationNode <= num_nodes; destinationNode++){
                if(destinationNode == startNode) continue;
                /*if(graph[destinationNode] == null) {
                	if(destinationNode == num_nodes) System.out.println(-1);
                	else System.out.println(-1 + " ");
                }*/
                boolean[] visited = new boolean[num_nodes+1];// to mark visited node for avoiding cycle.
                Deque<Integer> queue = new ArrayDeque<>();
                queue.addLast(startNode);
                visited[startNode] = true;
                int distance = 0;
                boolean gotPath = false;
                while(!queue.isEmpty()) {
                    int levelSize = queue.size();
                    for(int j = 0; j < levelSize; j++) {
                        int curNode = queue.removeFirst();
                        if(curNode == destinationNode){
                            result.add(distance);
                            gotPath = true;
                            break;
                        }else{
                                for(int k = 0;graph[curNode] != null && k < graph[curNode].size(); k++){
                                	int nextNode = (int)graph[curNode].get(k); // why need we cast to int?
                                	if(!visited[nextNode]){
                                		queue.addLast(nextNode);
                                		visited[nextNode] = true;
                                	}
                                }
                            }
                        }
                    distance += 6;
                    }
                	if(!gotPath) result.add(-1);
                }
	        	for(int n = 0; n < result.size()-1; n++) {
	        		System.out.print(result.get(n) + " ");
	        	}
	        	System.out.print(result.get(result.size()-1));
	            System.out.println();
            }
            System.out.println();
        }

}