Category Archives: Example

How to Mask JSON Confidential/Personal Information in logs :JAVA

Here you will see all steps to mask confidential/ information like credit card, CVV, Exp date,  SSN, password etc. So that it will print in mask form as ****** so that unauthorize use will not misuse of others information.

Here using Google GSON and GsonBuilder converting Java object to JSON and again converting JSON to Java Object.

By using Java refelection api’s replacing SPI fields data with *******.

Input JSON File AccountDetail


{
  "firstName": "Saurabh",
  "lastName": "Gupta",
  "address": {
    "addressLine1": "Noida City Center",
    "city": "Noida",
    "state": "UP",
    "pincode": "India",
    "country": "20310"
  },
  "creditCardDetail": {
    "cardNumber": "1234567890123456",
    "cvv": "123",
    "expDate": "12/90"
  }
}

 

Code to Mask JSON

package com.mask.json;

import java.io.IOException;
import java.lang.reflect.Field;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;

public class MaskJSONTest {
	static Set fieldSet = new HashSet();
	static List fieldNames = Arrays.asList("cardNumber", "cvv", "expDate");

	public static void main(String[] args) throws IOException {
		StringBuilder contentBuilder = new StringBuilder();
		try (Stream stream = Files.lines(
				Paths.get(
						"D:\\Saurabh Gupta\\Workspace\\JavaTestExamples\\src\\main\\resources\\AccountDetail.json"),
				StandardCharsets.UTF_8)) {
			stream.forEach(s -> contentBuilder.append(s).append("\n"));
		} catch (IOException e) {
			e.printStackTrace();
		}
		//Create GSON object
		//apply NullSearialization and Pretty formatting by GSON Builder
		Gson gson = getJsonBuilder().create();
		AccountDetail accounDetail = gson.fromJson(contentBuilder.toString(), AccountDetail.class);
		mask(accounDetail);
		System.out.println(gson.toJson(accounDetail));
	}
	public static GsonBuilder getJsonBuilder()
	{
		GsonBuilder builder = new GsonBuilder();

		// Setting for formatted output and serialize null value
		builder.setPrettyPrinting().serializeNulls();

		return builder;
	}

	public static void mask(Object object) {
		try {
			Field[] fields = object.getClass().getDeclaredFields();
			Object value = null;
			for (int i = 0; i <span id="mce_SELREST_start" style="overflow:hidden;line-height:0;"></span>&lt; fields.length; i++) {
				fields[i].setAccessible(true);
				value = fields[i].get(object);
				if (value != null &amp;&amp; fieldSet.add(fields[i].getName())) {
					if (fields[i].getType().isArray()
							|| fields[i].getType().getCanonicalName().startsWith(&quot;com.mask.json&quot;)) {
						mask(value);
					} else {
						if (fieldNames.contains(fields[i].getName()) &amp;&amp; fields[i].get(object) != null) {
							fields[i].set(object, replaceDigits((String) fields[i].get(object)));
						}
					}
				}

			}
		} catch (IllegalAccessException ex) {
			ex.printStackTrace();
		}
	}

	private static String replaceDigits(String text) {
		StringBuffer buffer = new StringBuffer(text.length());
		Pattern pattern = Pattern.compile(&quot;\\d&quot;);
		Matcher matcher = pattern.matcher(text);
		while (matcher.find()) {
			matcher.appendReplacement(buffer, &quot;X&quot;);
		}
		return buffer.toString();
	}

}

Model Classes


package com.mask.json;

public class AccountDetail {
private String firstName;
private String lastName;
private AddressDetail address;
private CreditCardDetail creditCardDetail;
public AccountDetail(String firstName, String lastName, AddressDetail address, CreditCardDetail creditCardDetail) {
	super();
	this.firstName = firstName;
	this.lastName = lastName;
	this.address = address;
	this.creditCardDetail = creditCardDetail;
}
public String getFirstName() {
	return firstName;
}
public void setFirstName(String firstName) {
	this.firstName = firstName;
}
public String getLastName() {
	return lastName;
}
public void setLastName(String lastName) {
	this.lastName = lastName;
}
public AddressDetail getAddress() {
	return address;
}
public void setAddress(AddressDetail address) {
	this.address = address;
}
public CreditCardDetail getCreditCardDetail() {
	return creditCardDetail;
}
public void setCreditCardDetail(CreditCardDetail creditCardDetail) {
	this.creditCardDetail = creditCardDetail;
}

}


package com.mask.json;

public class AddressDetail {

private String addressLine1;
private String city;
private String state;
private String pincode;
private String country;

public AddressDetail(String addressLine1, String city, String state, String pincode, String country) {
	super();
	this.addressLine1 = addressLine1;
	this.city = city;
	this.state = state;
	this.pincode = pincode;
	this.country = country;
}
public String getAddressLine1() {
	return addressLine1;
}
public void setAddressLine1(String addressLine1) {
	this.addressLine1 = addressLine1;
}
public String getCity() {
	return city;
}
public void setCity(String city) {
	this.city = city;
}
public String getState() {
	return state;
}
public void setState(String state) {
	this.state = state;
}
public String getPincode() {
	return pincode;
}
public void setPincode(String pincode) {
	this.pincode = pincode;
}
public String getCountry() {
	return country;
}
public void setCountry(String country) {
	this.country = country;
}

}

package com.mask.json;

public class CreditCardDetail{
private String cardNumber;
private String cvv;
private String expDate;

public CreditCardDetail(String cardNumber, String cvv, String expDate) {
	super();
	this.cardNumber = cardNumber;
	this.cvv = cvv;
	this.expDate = expDate;
}

public String getCardNumber() {
	return cardNumber;
}
public void setCardNumber(String cardNumber) {
	this.cardNumber = cardNumber;
}
public String getCvv() {
	return cvv;
}
public void setCvv(String cvv) {
	this.cvv = cvv;
}
public String getExpDate() {
	return expDate;
}
public void setExpDate(String expDate) {
	this.expDate = expDate;
}

}

Output Masked JSON :

{
“firstName”: “Saurabh”,
“lastName”: “Gupta”,
“address”: {
“addressLine1”: “Noida City Center”,
“city”: “Noida”,
“state”: “UP”,
“pincode”: “India”,
“country”: “20310”
},
“creditCardDetail”: {
“cardNumber”: “XXXXXXXXXXXXXXXX”,
“cvv”: “XXX”,
“expDate”: “XX/XX”
}
}

Related Posts

Below are some more masking ways for different type of data like XML, JSON and printing objects before logging , sending to page or transferring over network.

Log4j2: How to Mask Logs Personal/Confidential/SPI Information

How to MASK XML Confidential/Personal Data : JAVA

How to mask JAVA Object confidential/personal information in logs while Printing

Advertisements

[Solved] java.lang.IllegalStateException: No match found : Regular Expression

java.lang.IllegalStateException is most common Runtime Exception and Unchecked Exception . It throws by JVM when trying to access object methods which are not allowed on certain cases.

In below example matcher.group(1) throwing IllegalStateException as “not match” . In below case as java.lang.Regex first we should call to matcher.find() method to get match in text for specific regular expression then based on matcher.group() we can get match text.

package com.test.exceptions;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class RegularExpressionTest {

	public static void main(String[] args) {
		String text = "Facing Issues on IT";
		Pattern pattern = Pattern.compile("\\w");
		Matcher matcher = pattern.matcher(text);
		if (matcher != null) {
			System.out.println(matcher.group(1));//Exception on this point
			System.out.println("Match Count :" + matcher.groupCount() +matcher.find());
		}
	}
}

Output

Exception in thread "main" java.lang.IllegalStateException: No match found
	at java.util.regex.Matcher.group(Unknown Source)
	at com.test.exceptions.RegularExpressionTest.main(RegularExpressionTest.java:13)

Solution 

In below code match.find() not called before matcher.group(1) that’s why throwing IllegalStateException.

Summary

  • Explain about IllegalStateException and Example
  • For Regular Expression in java.lanag.Regex what need to take care.
  • How to solve it.

 

[Solved] CloneNotSupportedException

java.lang.CloneNotSupportedException throws when object’s class does not implement the cloneable interface and  clone method in class Object has been called to clone an object.
Some applications that override the clone method can also throw this exception to indicate that an object could not or should not be cloned.

Cloneable is marker interface which is not having any method to implement. When implement marker interface in class it indicate to JVM how to treat with this objects.
For more info about marker interface , it’s type and how to create check below link.

Marker Interface in Java, Use and Custom Marker Interface

clone() is protected method of object class which is super class of all the classes. if need to use clone() method have to override it .

Constructors :

  • CloneNotSupportedException()

Create a CloneNotSupportedException object with no detail message.

  • CloneNotSupportedException(String message)

Create a CloneNotSupportedException object with  exception detail message.

Example

In below example creating clone object of Employee class from object there is no any compile time error because we have override clone method of Object Class in Employee class but it will throw CloneNotSupportedException while execution because as per

Contract : if cloneable interface is implemented then only clone method will call otherwise  JVM will throw CloneNotSupportedException.

Here we are doing Shallow cloning of object. For more info on shallow and deep cloning follow below link.

Employee Class

public class Employee {
private String firstName;
private String lastName;
private int age;
private double salary;
private Address address;

public Employee(String firstName, String lastName, int age, double salary, Address address) {
	super();
	this.firstName = firstName;
	this.lastName = lastName;
	this.age = age;
	this.salary = salary;
	this.address = address;
}
public String getFirstName() {
	return firstName;
}
public void setFirstName(String firstName) {
	this.firstName = firstName;
}
public String getLastName() {
	return lastName;
}
public void setLastName(String lastName) {
	this.lastName = lastName;
}
public int getAge() {
	return age;
}
public void setAge(int age) {
	this.age = age;
}
public double getSalary() {
	return salary;
}
public void setSalary(double salary) {
	this.salary = salary;
}
public Address getAddress() {
	return address;
}
public void setAddress(Address address) {
	this.address = address;
}

@Override
public String toString() {
	return "Employee [firstName=" + firstName + ", lastName=" + lastName + ", age=" + age + ", salary=" + salary
			+ ", address=" + address + "]";
}
@Override
protected Object clone() throws CloneNotSupportedException {
	return super.clone();
}

}


Address Class


public class Address {
	private String addressLine1;
	private String city;
	private String state;
	private String contry;
	private String pincode;

	public Address(String addressLine1, String city, String state, String contry, String pincode) {
		super();
		this.addressLine1 = addressLine1;
		this.city = city;
		this.state = state;
		this.contry = contry;
		this.pincode = pincode;
	}

	public String getAddressLine1() {
		return addressLine1;
	}

	public void setAddressLine1(String addressLine1) {
		this.addressLine1 = addressLine1;
	}

	public String getCity() {
		return city;
	}

	public void setCity(String city) {
		this.city = city;
	}

	public String getState() {
		return state;
	}

	public void setState(String state) {
		this.state = state;
	}

	public String getContry() {
		return contry;
	}

	public void setContry(String contry) {
		this.contry = contry;
	}

	public String getPincode() {
		return pincode;
	}

	public void setPincode(String pincode) {
		this.pincode = pincode;
	}

	@Override
	public String toString() {
		return "Address [addressLine1=" + addressLine1 + ", city=" + city + ", state=" + state + ", contry=" + contry
				+ ", pincode=" + pincode + "]";
	}

}


Test for ClassNotSupportedException


public class CloneExample {

	public static void main(String[] args) {
		Address address = new Address("Next to Metro Station", "Noida", "UP", "India", "201301");
		Employee employee = new Employee("Saurabh", "Gupta", 30, 50000, address);
		System.out.println("Employee Object Before Clone :");
		System.out.println(employee);
		try {
                        System.out.println("Employee Object After Clone :");
			Employee employeeCopy = (Employee) employee.clone();

			System.out.println(employeeCopy);
		} catch (CloneNotSupportedException ex) {
			ex.printStackTrace();
		}

	}

}


Output

Employee Object Before Clone :
Employee [firstName=Saurabh, lastName=Gupta, age=30, salary=50000.0, address=Add
ress [addressLine1=Next to Metro Station, city=Noida, state=UP, contry=India, pi
ncode=201301]]
Employee Object After Clone :
java.lang.CloneNotSupportedException: Employee
	at java.lang.Object.clone(Native Method)
	at Employee.clone(Employee.java:55)
	at CloneExample.main(CloneExample.java:11)

Solutions :

To solve this CloneNotSupportedException we have to implement cloneable interface in Employee class as below and run this program again.

public class Employee implements Cloneable<span id="mce_SELREST_start" style="overflow:hidden;line-height:0;"></span>{

}

Summary :

  • Explain about CloneNotSupportedException by example and how it happen.
  • Constructors of CloneNotSupportedException.
  • How to fix CloneNotSupportedException.
  • Cloneable Marker Interface.

References :

https://docs.oracle.com/javase/8/docs/api/java/lang/CloneNotSupportedException.html

Marker Interface in Java and Use

Marker interface is main frequently asked question to JAVA interview. Generally interviewer asked this question to check internal know knowledge of JAVA developer some times ask on architect level also because it’s follow the Marker Interface Design pattern. Here I have covered everything to crack questions related to Marker Interface.

What is a Marker Interface?

Marker Interfaces are empty interface it does not contains any properties and behaviors to implement. It’s also called as marker or the tag interface.

Why Marker Interface and Who Implement it?

Custom Marker Interface Example
Custom Marker Interface

Marker interface functionality is predefined implicitly in JVM. When a class implement a marker interface then class is not expected to implement anything to adhere to the contract defined by the interface. In contrast it is a tag to indicate the JVM regarding an expected functionality to perform implicitly.

For Example :

java.io.Serializable : If class implements the Serializable interface then JVM perform some special operation on it and writes the state of the object into object stream then this object stream will available to read by another JVM.

java.lang.Cloneable: Same way if class is implementing Cloneable interface then it perform some special operation to clone object by copy all fields

What are available Marker Interfaces in JAVA?

Mainly used built-in marker interfaces are as below :

  • java.lang.Cloneable
  • java.io.Serializable
  • java.rmi.Remote
  • java.util.EventListener (its officially know as ‘tagging interface’)

There are some more marker interface present in Java.

  • java.util.concurrent.CompletableFuture.AsynchronousCompletionTask
  • java.sql.ParameterMetaData
  • javax.xml.stream.events.EndDocument
  • javax.management.loading.PrivateClassLoader
  • java.security.KeyStore.Entry
  • java.security.KeyStore.LoadStoreParameter
  • java.security.KeyStore.ProtectionParameter
  • java.security.Policy.Parameters
  • javax.security.auth.callback.Callback
  • javax.security.auth.login.Configuration.Parameter

Can we create custom marker interfaces ?

Yes

How to Create Custom marker Interfaces?

Custom Marker Interface is nothing to do specially with JVM end . It’s to mentioned in class methods to treat this object specially so that perform some special operations.

Steps to create marker interface :

  • Create an interface with no properties and method on it.
  • Implements this interface on class.
  • Perform any operation in method if object instance of type marker interface.

See below example for more detail

Create Empty  Interface

public interface MyMarker {
//no properties and method
}

Write a class to implement interface

public class MyClass implements MyMarker{
//define properties and method
}

Class to check Marker Interface to perform special operation

public class TestMarker {

	public static void main(String[] args) {
		MyClass myClass=new MyClass();

		if(myClass instanceof MyMarker)
		{
			System.out.println("I am special treat me VIP");
		}

	}

}

Output :

I am special treat me VIP

Summary

  • Explained about Marker Interface and how JVM handle Marker Interface.
  • Available Marker Interface in Java with some example.
  • Creation of Custom Marker Interface and use.

 

[Solved] ClassNotFoundException: com.mysql.jdbc.Driver

java.lang.ClassNotFoundException is Checked Exception which is subclass of  java.lang.ReflectiveOperationException. This  is thrown when application load a class by String name whose definition is not found. This exception occurs by below methods :

  • The forName method in the class Class.

                 Class.forName(java.lang.String)

  • The findSystemClass method in the class ClassLoader.

                ClassLoader.findSystemClass()

  • The loadClass method in class ClassLoader.

                ClassLoader.loadClass(java.lang.String, boolean)

 Difference between Class.forName() and ClassLoader.loadClass()

Example 1: ClassNotFoundException

Below is example of connecting with database and retrieve data from sql table. This will throw ClassNotFoundException because sql driver jar is not in classpath. After this example also mentioned solution.

package example;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;

public class ClassNotFoundException1 {

	public static void main(String[] args) {
		try {
			Class.forName("com.mysql.jdbc.Driver");

			Connection con = DriverManager.getConnection("jdbc:mysql://localhost:3306/university", "root", "");

			Statement stmt = con.createStatement();
			ResultSet rs = stmt.executeQuery("select name from employee");
			String dbtime;
			while (rs.next()) {
				dbtime = rs.getString(1);
				System.out.println(dbtime);
			}

			con.close();

		} catch (ClassNotFoundException | SQLException e) {
			System.out.println("Connection Failed! Check output console");
			e.printStackTrace();
		}

	}

}

Output:

Connection Failed! Check output console
java.lang.ClassNotFoundException: com.mysql.jdbc.Driver
	at java.net.URLClassLoader.findClass(Unknown Source)
	at java.lang.ClassLoader.loadClass(Unknown Source)
	at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source)
	at java.lang.ClassLoader.loadClass(Unknown Source)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Unknown Source)
	at example.ClassNotFoundException1.main(ClassNotFoundException1.java:13)

Solutions:

For solving ClassNotFoundException by Class.ForName() method  considering above example to load MySQL driver same way you can follow for other classes and different DB drivers.

Solution in Eclipse :Follow below steps :

  • Right click your project folder and open up Properties.
  • From the right panel, select Java Build Path then go to Libraries tab.
  • Select Add External JARs to import the MySql driver.
  • From the right panel, select Deployment Assembly.
  • Select Add…, then select Java Build Path Entries and click Next.
  • You should see the SQL driver on the list. Select it and click first.

Tomcat :

If directly running from tomcat.Just copy the MySql-Connector.jar into Tomcat’s lib folder/directory, and then remove the jar from the webApp’s lib folder, and then, run the project.

Summary :

  • Define what is ClassNotFoundException.
  • What are methods throws ClassNotFoundException?
  • Example for ClassNotFoundException.
  • How to fix ClassNotFoundException in Eclipse and Tomcat.

[Solved] ClassCastException :A cannot be cast to B : Example

java.lang.ClassCastException is RuntimeException and Unchecked Exception which throws when code has attempted to cast an object to a subclass or class of which it is not an instance.

How to Fix ClassCastException ?

  • Always careful when try to cast an object of a class into another class. Make sure that new  type class belongs to one of its parent classes.
  • Use Generics to prevent ClassCastException because Generics provide compile time checks to develop type-safe applications so that issue identified on compile time.

Note: The conversion is valid only in cases where a class extends a parent class and the child class is casted to its parent class.

Example :
Below are two example of ClassCastException where type casting an objet to different type which is not sub class of object class.

Example 1: Here first converting Integer Object to Object class which is parent class of all classes and then after type casting object of Object class to String Object. Which not compatible with Integer class and not sub class of Integer that’s why throwing ClassCastException.

import java.util.Arrays;
import java.util.Collections;
import java.util.List;

public class ClassCastExceptionExample {

	public static void main(String[] args) {
		/**
		 * when one tries to cast an Integer to a String, String is not an subclass of Integer, so a ClassCastException will be thrown.
	 */
		try
		{
		Object object= new Integer(10);
		System.out.println((String)object);
		}
		catch(ClassCastException ex)
		{
			ex.printStackTrace();
		}
	}

}

Example 2:
Here we  are trying to type cast an object of class A into an object of class B, and they aren’t compatible, we will get a class cast exception.

Let’s think of a collection of classes from below example

class A {}
class B extends A {…}
class C extends A {…}
We can cast any of these things to Object, because all Java classes inherit from Object.
We can cast either B or C to A, because they’re both “kinds of” A
We can cast a reference to an A object to B only if the real object is a B.
We can’t cast a B to a C even though they’re both A’s on that point will throw type cast exception.

public class A
{
}
public class B extends A
{
}
public class C extends A
{
}

package example;

public class ClassCastExceptionExample {

	public static void main(String[] args) {		

		try {
         A a=new A();
         A b=new B();
         A c=new C();

         B d=(B)b;
         B e=(B)c;// Exception on that point

		} catch (ClassCastException ex) {
			ex.printStackTrace();
		}
	}

}

Output:

java.lang.ClassCastException: example.C cannot be cast to example.B
	at example.ClassCastExceptionExample.main(ClassCastExceptionExample.java:27)

[Solved] Exception UnsupportedOperationException Example

java.lang.UnsupportedOperationException is RuntimeException and Unchecked Exception which expected to thrown by JVM(Java Virtual Machine) when try to perform an “optional operation” which is not allowed an object .

The Java framework contains plenty of these, especially in the Collections framework. For example “add” is an optional operation, because immutable collections should not allow it. Throwing UnsupportedOperationException is exactly what you should do if you don’t want to write one of these methods.

Constructors :

  • UnsupportedOprationException() : Constructs an UnsupportedOperationException with no detail message.
  • UnsupportedOprationException(String message) : Constructs an UnsupportedOperationException with specified detail message.
  • UnsupportedOprationException(String message, Throwable cause) : Constructs an UnsupportedOperationException with specified detail message and cause.
  • UnsupportedOprationException(Throwable cause) : Constructs a new exception with the  specified cause and a detail message of (cause==null ? null : cause.toString()) (which typically contains the class and detail message of cause).

Example :

In below example problem is List returned from Arrays.AsList() is not like java.util.ArrayList. Arrays.asList() returns a java.util.Arrays$ArrayList which is an immutable list. You can not add or remove any element from this list otherwise will get java.lang.UnsupportedOperationException Exception.

package exceptionhandeling;

import java.util.Arrays;
import java.util.List;

public class UnsupportedOperationException {

	public static void main(String[] args) {
		String [] empArr={"Saurabh","Gaurav","Shailesh","Ankur","Ranjith","Ramesh"};
		//Convert Array to LIst
		List empList=Arrays.asList(empArr);
		/**
		 * By the time you get to the remove(i) statement, list is no longer a java.util.ArrayList.
		 * When you call Arrays.asList it does not return a java.util.ArrayList. It returns a java.util.Arrays$ArrayList which is an immutable list. You cannot add to it and you cannot remove from it.

		 * Not every List implementation support add method because Arrays.asList() returned
		 * immutable list of fixed size
		 */
		for(String emp:empList)
		{
			empList.add("Sachin");
			System.out.println(emp);
		}

	}

}

Output:

Exception in thread "main" java.lang.UnsupportedOperationException
	at java.util.AbstractList.add(Unknown Source)
	at java.util.AbstractList.add(Unknown Source)
	at example.UnsupportedOperationExceptionExample.main(UnsupportedOperationExceptionExample.java:21)

Java : Comparable Vs Comparator

Comparable and Comparator both are interface used for compare and sorting collections of objects. Where  Comparable used for natural ordering of objects while Comparator used for sorting on user specific value on different type objects. In both the cases if objects are not equate or compatible will through ClassCastException.

In below links, I have explained about in depth detail about Comparable and Comparator implementation and cases where to use. Here mainly focus on some points for technical, implementation and functional points.

Comprable :Sort ArrayList in Ascending or Descending Order or Natural or Chronological Order

How to Sort By Comparable Interface in Ascending and Descending Order : Java

How to sort object by Comparator interface in ascending and descending order : JAVA

Difference between Comparable and Comparator

Comparable Comparator
Comparable is in java lang package as java.lang.Comparable. Comparator is in java util package as java.util.Comparator.
Method used int compareTo(Object t); Method used int compare(Object t1 , Object t2);

Comparable is used to compare itself by using with another object.

Here modify the class whose instance you want to sort. So that only one sort sequence can be created per class.

Comparator is used to compare two datatypes are objects.

Here build a class separate from class whose instance you want to sort. So that multiple sort sequence can be created per class.

 A comparable used for default and natural ordering of objects.  A comparator represents the ordering itself for specific use.

Some java classes having build in Comparable interface like String, Date, Wrapper Classes etc.

Some classes actually provide Comparators for common cases; for instance, Strings are by default case-sensitive when sorted, but there is also a static Comparator called CASE_INSENSITIVE_ORDER.
 For sorting use methods like Collections.Sort(List) or Array.sort().  For Sorting use method Collections.sort(List, Comparator).

Reference :

https://docs.oracle.com/javase/tutorial/collections/interfaces/order.html

How to sort object by Comparator interface in ascending and descending order : JAVA

As in previous posts shown like wrapper of primitive type , String and Date classes having  implements in-built comparable interface and also sort user-defined classes by implements  Comparable interface which provide natural or chronological or alphabetical sorting of elements.

Sort ArrayList in Ascending or Descending Order or Natural or Chronological Order

How to Sort By Comparable Interface in Ascending and Descending Order : Java

Java : Comparable Vs Comparator

What is Comparator Interface?

java.util.Comparator  interface imposes user define sorting on different class fields  based on comparing values from different objects.  It provide compare() method which compares two objects  and returns a negative integer, 0, or a positive integer depending on whether the receiving object is less than, equal to, or greater than the specified object. If the specified object cannot be compared to the receiving object, the method throws a ClassCastException.

How to use Comparator Interface?

Below is syntax of compare method:

public interface Comparator {
    public int compare(T o1, T o2);

}

compare() method compare two objects  fields values and return negative integer, 0 or a positive integer and Collections.sort() method will sort based on this return value.

Lists (and arrays) of objects that implement this comparator interface can be sorted automatically by Collections.sort (and Arrays.sort). Objects that implement this comparator  interface can be used as keys in a sorted map or as elements in a sorted set, without the need to specify a comparable.

Note :  Below example is base on List of Employee objects it cannot be used to order a sorted collection, such as TreeSet because it generates an ordering that is not compatible with equals. It means that Comparator equates objects that the equals method does not.

In particular, any two employees who were joined on the same date will compare as equal. When you’re sorting a List this doesn’t matter; but when you’re using the Comparator to order a sorted collection, it’s fatal. If you use this Comparator to insert multiple employees joined on the same date into a TreeSet only the first one will be added to the set; the second will be seen as a duplicate element and will be ignored.

Example :

Below is simple Employee class which is having fields and getter/setter methods and also constructor to create object.

package sorting
public class Employee {
private int id;
private String firtsName;
private String lastName;
private String designation;
private double salary;
private int age;

//Default Constructor
public Employee()
{
}
//Parametrize Constructor
public Employee(int id, String firtsName, String lastName, String designation, double salary, int age) {
	super();
	this.id = id;
	this.firtsName = firtsName;
	this.lastName = lastName;
	this.designation = designation;
	this.salary = salary;
	this.age = age;
}

public int getId() {
	return id;
}
public void setId(int id) {
	this.id = id;
}
public String getFirtsName() {
	return firtsName;
}
public void setFirtsName(String firtsName) {
	this.firtsName = firtsName;
}
public String getLastName() {
	return lastName;
}
public void setLastName(String lastName) {
	this.lastName = lastName;
}
public String getDesignation() {
	return designation;
}
public void setDesignation(String designation) {
	this.designation = designation;
}
public double getSalary() {
	return salary;
}
public void setSalary(double salary) {
	this.salary = salary;
}
public int getAge() {
	return age;
}
public void setAge(int age) {
	this.age = age;
}

@Override
public String toString() {
	return "Employee [id=" + id + ", firtsName=" + firtsName + ", lastName=" + lastName + ", designation=" + designation
			+ ", salary=" + salary + ", age=" + age + "]";
}
}

Here we have created EmpComparator class and implements compare() method  of java.util.Comparator interface which will compare object fields values based on selected fields on time of sorting. ex: sortingField value.

For example : sortingField value as firstName then compare() method of  java.util.Comparator  interface will compare firstName value of passing two object on compare() method and return a integer value like (negative integer, 0 or positive integer) depend on compare result.

package sorting;
import java.util.Comparator;

public class EmpComparator implements Comparator {
	public static final String ID = "id";
	public static final String FIRST_NAME = "firstName";
	public static final String LAST_NAME = "lastName";
	public static final String DESIGNATION = "designation";
	public static final String SALARY = "salary";
	public static final String AGE = "age";
	private String sortingField;

	@Override
	public int compare(Employee emp1, Employee emp2) {
		int diff = 0;
		switch (sortingField) {
		case ID:
			diff=emp1.getId()-emp2.getId();
			break;
		case FIRST_NAME:
			diff=emp1.getFirtsName().compareTo(emp2.getFirtsName());
			break;
		case LAST_NAME:
			diff=emp1.getLastName().compareTo(emp2.getLastName());
			break;
		case DESIGNATION:
			diff=emp1.getDesignation().compareTo(emp2.getDesignation());
			break;
		case SALARY:
			diff=(int)(emp1.getSalary()-emp2.getSalary());
			break;
		case AGE:
			diff=emp1.getAge()-emp2.getAge();
			break;

		}
		return diff;
	}

	public String getSortingField() {
		return sortingField;
	}

	public void setSortingField(String sortingField) {
		this.sortingField = sortingField;
	}
}

Here in below example creating List of Employee objects for sorting in ascending and descending order.

For sorting by firstName assigning value of sortingField as “firstName” in empComparator object. First we are using Collections.sort(List, Comparator) sorting method to sort list in ascending order by firstName because current value of sortingField is firstName. For descending order we will just reverse order of objects by Collections.resverse() method. Similarly using for lastName and also try with other fields of Employee class.

package sorting;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

public class SortComparator {

	public static void main(String[] args) {
		Employee[] empArr = {
				new Employee(1, "Saurabh", "Gupta", "Sr Project Lead", 60000, 35),
				new Employee(2, "Gaurav", "Gupta", "Developer", 50000, 32),
				new Employee(3, "Shailesh", "Nagar", "Manager", 100000, 36),
				new Employee(4, "Ankur", "Mehrotra", "Lead", 55000, 30),
				new Employee(5, "Ranjith", "Ranjan", "Tester", 35000, 45),
				new Employee(6, "Ramesh", "Bhardwaj", "Support", 25000, 35)
				};
		// Convert Array to LIst
		List empList = Arrays.asList(empArr);
		// Print Assigned Values Before Sort;
		System.out.println("********Print Employee List Before Sort********");
		printArrayList(empList);

		// Sort List in Ascending order by first Name
		EmpComparator empComparator = new EmpComparator();
		empComparator.setSortingField("firstName");
		Collections.sort(empList, empComparator);
		System.out.println("\n********Print Employee List in Ascending Order********");
		printArrayList(empList);

		// Sort List in Descending order by FirtName
		Collections.reverse(empList);
		System.out.println("\n********Print Employee List in Descending Order********");
		printArrayList(empList);

		// Sort List in Ascending order by lastName

		empComparator.setSortingField("lastName");
		Collections.sort(empList, empComparator);
		System.out.println("\n********Print Employee List in Ascending Order********");
		printArrayList(empList);

		// Sort List in Descending order by last Name
		Collections.reverse(empList);
		System.out.println("\n********Print Employee List in Descending Order********");
		printArrayList(empList);

	}

	private static void printArrayList(List empList) {
		for (Employee emp : empList) {
			System.out.println(emp);<span id="mce_SELREST_start" style="overflow:hidden;line-height:0;"></span>
		}
	}
}

Output :

********Print Employee List Before Sort********
Employee [id=1, firtsName=Saurabh, lastName=Gupta, designation=Sr Project Lead, salary=60000.0, age=35]
Employee [id=2, firtsName=Gaurav, lastName=Gupta, designation=Developer, salary=50000.0, age=32]
Employee [id=3, firtsName=Shailesh, lastName=Nagar, designation=Manager, salary=100000.0, age=36]
Employee [id=4, firtsName=Ankur, lastName=Mehrotra, designation=Lead, salary=55000.0, age=30]
Employee [id=5, firtsName=Ranjith, lastName=Ranjan, designation=Tester, salary=35000.0, age=45]
Employee [id=6, firtsName=Ramesh, lastName=Bhardwaj, designation=Support, salary=25000.0, age=35]

********Print Employee List in Ascending Order********
Employee [id=4, firtsName=Ankur, lastName=Mehrotra, designation=Lead, salary=55000.0, age=30]
Employee [id=2, firtsName=Gaurav, lastName=Gupta, designation=Developer, salary=50000.0, age=32]
Employee [id=6, firtsName=Ramesh, lastName=Bhardwaj, designation=Support, salary=25000.0, age=35]
Employee [id=5, firtsName=Ranjith, lastName=Ranjan, designation=Tester, salary=35000.0, age=45]
Employee [id=1, firtsName=Saurabh, lastName=Gupta, designation=Sr Project Lead, salary=60000.0, age=35]
Employee [id=3, firtsName=Shailesh, lastName=Nagar, designation=Manager, salary=100000.0, age=36]

********Print Employee List in Descending Order********
Employee [id=3, firtsName=Shailesh, lastName=Nagar, designation=Manager, salary=100000.0, age=36]
Employee [id=1, firtsName=Saurabh, lastName=Gupta, designation=Sr Project Lead, salary=60000.0, age=35]
Employee [id=5, firtsName=Ranjith, lastName=Ranjan, designation=Tester, salary=35000.0, age=45]
Employee [id=6, firtsName=Ramesh, lastName=Bhardwaj, designation=Support, salary=25000.0, age=35]
Employee [id=2, firtsName=Gaurav, lastName=Gupta, designation=Developer, salary=50000.0, age=32]
Employee [id=4, firtsName=Ankur, lastName=Mehrotra, designation=Lead, salary=55000.0, age=30]

********Print Employee List in Ascending Order********
Employee [id=6, firtsName=Ramesh, lastName=Bhardwaj, designation=Support, salary=25000.0, age=35]
Employee [id=1, firtsName=Saurabh, lastName=Gupta, designation=Sr Project Lead, salary=60000.0, age=35]
Employee [id=2, firtsName=Gaurav, lastName=Gupta, designation=Developer, salary=50000.0, age=32]
Employee [id=4, firtsName=Ankur, lastName=Mehrotra, designation=Lead, salary=55000.0, age=30]
Employee [id=3, firtsName=Shailesh, lastName=Nagar, designation=Manager, salary=100000.0, age=36]
Employee [id=5, firtsName=Ranjith, lastName=Ranjan, designation=Tester, salary=35000.0, age=45]

********Print Employee List in Descending Order********
Employee [id=5, firtsName=Ranjith, lastName=Ranjan, designation=Tester, salary=35000.0, age=45]
Employee [id=3, firtsName=Shailesh, lastName=Nagar, designation=Manager, salary=100000.0, age=36]
Employee [id=4, firtsName=Ankur, lastName=Mehrotra, designation=Lead, salary=55000.0, age=30]
Employee [id=2, firtsName=Gaurav, lastName=Gupta, designation=Developer, salary=50000.0, age=32]
Employee [id=1, firtsName=Saurabh, lastName=Gupta, designation=Sr Project Lead, salary=60000.0, age=35]
Employee [id=6, firtsName=Ramesh, lastName=Bhardwaj, designation=Support, salary=25000.0, age=35]
<span id="mce_SELREST_start" style="overflow:hidden;line-height:0;"></span>

Reference :

https://docs.oracle.com/javase/tutorial/collections/interfaces/order.html

How to Sort By Comparable Interface in Ascending and Descending Order : Java

As in previous post shown like wrapper of primitive type , String and Date classes having  implements in-built comparable interface which provide natural or chronological or alphabetical sorting of elements.

Sort ArrayList in Ascending or Descending Order or Natural or Chronological Order

How to sort object by Comparator interface in ascending and descending order : JAVA

Java : Comparable Vs Comparator

What is Comparable Interface?

java.lang.Comparable  interface imposes a total natural ordering on the objects of each class that implements it.  it provide compareTo() method which compares the receiving object with the specified object and returns a negative integer, 0, or a positive integer depending on whether the receiving object is less than, equal to, or greater than the specified object. If the specified object cannot be compared to the receiving object, the method throws a ClassCastException.

How to use Comparable Interface?

Below is syntax of compareTo method:

public interface Comparable {
    public int compareTo(T o);

}

compareTo() method compare current object (this)  fields values with passing object fields values and return negative integer, 0 or a positive integer and Collections.sort() method will sort based on this return value.

Lists (and arrays) of objects that implement this comparable interface can be sorted automatically by Collections.sort (and Arrays.sort). Objects that implement this comparable interface can be used as keys in a sorted map or as elements in a sorted set, without the need to specify a comparator.

Example :

Below Employee class implements compareTo() method of  java.lang.Comparable  interface which is comparing firstName value with specified object firstName value. This  method return a integer value like (negative integer, 0 or positive integer) depend on compare result.

package sorting;

public class Employee implements Comparable{
private int id;
private String firtsName;
private String lastName;
private String designation;
private double salary;
private int age;

//Default Constructor
public Employee()
{

}

//Parametrize Constructor
public Employee(int id, String firtsName, String lastName, String designation, double salary, int age) {
	super();
	this.id = id;
	this.firtsName = firtsName;
	this.lastName = lastName;
	this.designation = designation;
	this.salary = salary;
	this.age = age;
}

@Override
public int compareTo(Employee employee) {
	//sort by firstName
	return this.firtsName.compareTo(employee.firtsName);
}

public int getId() {
	return id;
}
public void setId(int id) {
	this.id = id;
}
public String getFirtsName() {
	return firtsName;
}
public void setFirtsName(String firtsName) {
	this.firtsName = firtsName;
}
public String getLastName() {
	return lastName;
}
public void setLastName(String lastName) {
	this.lastName = lastName;
}
public String getDesignation() {
	return designation;
}
public void setDesignation(String designation) {
	this.designation = designation;
}
public double getSalary() {
	return salary;
}
public void setSalary(double salary) {
	this.salary = salary;
}
public int getAge() {
	return age;
}
public void setAge(int age) {
	this.age = age;
}

@Override<span id="mce_SELREST_start" style="overflow:hidden;line-height:0;"></span>
public String toString() {
	return "Employee [id=" + id + ", firtsName=" + firtsName + ", lastName=" + lastName + ", designation=" + designation
			+ ", salary=" + salary + ", age=" + age + "]";
}

}

In below class sorting  Employee list objects by names in ascending order by Collections.sort(list). As shown above Employee Class implements  Comparable interface and it’s  compareTo() method  will sort elements in Ascending order by firstName on objects.

For Descending order used Collections.reverse(List) reverse method which will reverse list of sorted list.

 

package sorting;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

public class SortComparable {

	public static void main(String[] args) {
		Employee [] empArr={
				new Employee(1,"Saurabh","Gupta","Sr Project Lead",60000,35),
				new Employee(2,"Gaurav","Gupta","Developer",50000,32),
				new Employee(3,"Shailesh","Nagar","Manager",100000,36),
				new Employee(4,"Ankur","Mehrotra","Lead",55000,30),
				new Employee(5,"Ranjith","Ranjan","Tester",35000,45),
				new Employee(6,"Ramesh","Bhardwaj","Support",25000,35)
				};
		//Convert Array to LIst
		List empList=Arrays.asList(empArr);
		//Print Assigned Values Before Sort;
		System.out.println("********Print Employee List Before Sort********");
		printArrayList(empList);

		//Sort List in Ascending order by collections api
		Collections.sort(empList);
		System.out.println("\n********Print Employee List in Ascending Order********");
		printArrayList(empList);

		//Sort List in Descending order by collections api
		Collections.reverse(empList);
		System.out.println("\n********Print Employee List in Descending Order********");
		printArrayList(empList);

	}

	private static void printArrayList(List empList)
	{
		for(Employee emp:empList)
		{
			System.out.println(emp);
		}
	}
}

 

Reference :

https://docs.oracle.com/javase/tutorial/collections/interfaces/order.html

How to convert ArrayList to Array and Array to ArrayList?

Arrays class provide method Arrays.asList()  which convert array type elements to a unmodified fixed size ArrayList.

Same way List provide method as List.toArray(List) which convert ArrayList elements to Array.

Note : List returned from Arrays.AsList() is not like java.util.ArrayList. It
returns a java.util.Arrays$ArrayList which is an immutable list. You can not add or remove any element from this list otherwise will get java.lang.UnsupportedOperationException

Example : 

In below example used method Arrays.asList()  to convert  String array to ArrayList of String elements. On other  side use List.toArray() method to convert ArrayList elements to Arrays of type String elements.

package exceptionhandeling;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

public class ArrayToArrayListConversion {

	public static void main(String[] args) {
		String [] empArr={"Saurabh","Gaurav","Shailesh","Ankur","Ranjith","Ramesh"};
		//Convert Array to LIst
		List empList=Arrays.asList(empArr);
		System.out.println("*****Print ArrayList*****");
		printArrayList(empList);
		//Convert ArrayList to Array
		 empArr=(String[])empList.toArray();
		 System.out.println("*****Print Array*****");
		 printArray(empArr);
	}
	private static void printArrayList(List empList)
	{
		for(String emp:empList)
		{
			System.out.println(emp);
		}
	}
	private static void printArray(String[] empArr)
	{
		for(String emp:empArr)
		{
			System.out.println(emp);
		}
	}
}

Output:

*****Print ArrayList*****
Saurabh
Gaurav
Shailesh
Ankur
Ranjith
Ramesh
*****Print Array*****
Saurabh
Gaurav
Shailesh
Ankur
Ranjith
Ramesh

Sort ArrayList in Ascending or Descending Order or Natural or Chronological Order

Collections framework provide Collections.sort(List) method which sort String elements in natural or chronological order or ascending order.

For descending order use Collections.reverse(List) method to reverse order of complete list elements.

Related Topics:

How to Sort By Comparable Interface in Ascending and Descending Order : Java

How to sort object by Comparator interface in ascending and descending order : JAVA

Java : Comparable Vs Comparator

Points to Remember:

  • Below are classes  implements  Comparable interface which provide a natural ordering for a class  sorted automatically.
  • If sorting a list of the elements which do not  implement Comparable , Collections.sort(list) method will throw ClassCastException. Similarly, Collections.sort(list, comparator ) will throw ClassCastException if you try to sort a list whose elements cannot be compared to another using Comparator.
                                 Classes Implementing Comparable
Class Natural Ordering
Byte Signed numerical
Character Unsigned numerical
Long Signed numerical
Integer Signed numerical
Short Signed numerical
Double Signed numerical
Float Signed numerical
BigInteger Signed numerical
BigDecimal Signed numerical
Boolean Boolean.FALSE < Boolean.TRUE
File System-dependent lexicographic on path name
String Lexicographic
Date Chronological
CollationKey Locale-specific lexicographic

Example :
In below example sorting list of names in ascending order by Collections.sort(list). As shown above String Class by default having Comparable interface which will sort elements in Ascending or Chronological order.

For Descending order used Collections.reverse(List) reverse method which will reverse list of sorted list.

package sorting;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;

public class SortData {

	public static void main(String[] args) {
		String [] empArr={"Saurabh","Gaurav","Shailesh","Ankur","Ranjith","Ramesh"};
		//Convert Array to LIst
		List empList=Arrays.asList(empArr);
		//Print Assigned Values Before Sort;
		System.out.println("********Print Employee List Before Sort********");
		printArrayList(empList);

		//Sort List in Ascending order by collections api
		Collections.sort(empList);
		System.out.println("\n********Print Employee List in Ascending Order********");
		printArrayList(empList);

		//Sort List in Descending order by collections api
		Collections.reverse(empList);
		System.out.println("\n********Print Employee List in Descending Order********");
		printArrayList(empList);

	}

	private static void printArrayList(List empList)
	{
		for(String emp:empList)
		{
			System.out.println(emp);
		}
	}

}

Output:

********Print Employee List Before Sort********
Saurabh
Gaurav
Shailesh
Ankur
Ranjith
Ramesh

********Print Employee List in Ascending Order********
Ankur
Gaurav
Ramesh
Ranjith
Saurabh
Shailesh

********Print Employee List in Descending Order********
Shailesh
Saurabh
Ranjith
Ramesh
Gaurav
Ankur

Reference :

https://docs.oracle.com/javase/tutorial/collections/interfaces/order.html

[Solved] ArrayIndexOutOfBoundException in JAVA

java.lang.ArrayindexOutOfboundsException is Runtime and Unchecked Exception. It’s subclass of java.lang IndexOutOfBoundsException.

ArrayIndexOutOfBoundsException is most common error in Java Program. It throws when an array has been accessed with an illegal index either negative or greater than or equal to the size of the array.

Points To Remember:

  •  Array index starts at zero and goes to length – 1, for example in an integer array int[] counts= new int[20], the first index would be zero and last index out be 19 (20 -1)
  • Array index cannot be negative, hence counts[-1] will throw java.lang.ArrayIndexOutOfBoundsException.
  • The maximum array index can be Integer.MAX_VALUE -1 because array accept data type of index is int and max allowed value for int is Integer.MAX_VALUE.

Constructors:

  • ArrayIndexOutOfBoundsException() : Constructs an  ArrayIndexOutOfBoundsException with no detail message.
  • ArrayIndexOutOfBoundsException(int index) : Constructs a new ArrayIndexOutOfBoundsException class with an argument indicating the illegal index.
  • ArrayIndexOutOfBoundsException(String s) : Constructs an ArrayIndexOutOfBoundsException class with the specified detail message.

Example :

public class ArrayOutOfBoundException {

	public static void main(String[] args) {
		String [] empArr={"Saurabh","Gaurav","Shailesh","Ankur","Ranjith","Ramesh"};
		//No of employee in array
		//Will through IndexOuhtOfBoundException because array having only six element of index 0 to 5
		try
		{
		String name=empArr[8];
		System.out.println("Employee :"+empArr[8]);
		}
		catch(ArrayIndexOutOfBoundsException ex)
		{
			ex.printStackTrace();
		}
	}
}

Output:

java.lang.ArrayIndexOutOfBoundsException: 8
	at example.ArrayIndexOutOfBoundsException.main(ArrayIndexOutOfBoundsException.java:11)<span id="mce_SELREST_start" style="overflow:hidden;line-height:0;"></span>

[Solved] java.lang.ArithmeticException: / by zero in JAVA

java.lang.ArithmeticException is Unchecked exception and sub class of java.lang.RuntimeException. It’s thrown when an exceptional condition occurred in Arithmetic Operations. It can also occurred by virtual machine as if suppression were disabled and /or the stack trace was not writable.

Constructors:

  • ArithmeticException() : Constructs an ArithmeticException with no detail message.
  • ArithmeticException(String s) : Constructs an ArithmeticException with the specified detail message.
Example :
An integer value “divide by zero” throw ArithmaticException.  In below example i am dividing int, double, float and long value with 0. For long and int type value it’s throwing Arithmatic Exception while for double and float printing special value as Infinity.  See below How to fix Arithmetic Exception? section.
package example;

public class ArithmaticExceptionExample {

	public static void main(String[] args) {
		int x=0;
		int y=5;
		double z=6;
		float l=6;
		long k=10L;
		//Integer value divide by integer value as 0 throw ArithmeticException
		try
		{
			System.out.println("Integer value divide by zero");
		System.out.println(y/x);
		}
		catch(ArithmeticException ex)
		{
			ex.printStackTrace();
		}
		//Double value divide by integer value as 0 No Exception special value Infinity
		System.out.println("Double value divide by zero");
		System.out.println(z/x);

		//Float value divide by integer value as 0 No Exception special value Infinity
		System.out.println("Float value divide by zero");
		System.out.println(l/x);

		//Long value divide by integer value as 0 throw ArithmeticException
		try
		{
		System.out.println("Long value divide by zero");
		System.out.println(k/x);
		}
		catch(ArithmeticException ex)
		{
			ex.printStackTrace();
		}

	}

}

Output:

Integer value divide by zero
java.lang.ArithmeticException: / by zero
	at example.ArithmaticExceptionExample.main(ArithmaticExceptionExample.java:15)
Double value divide by zero
Infinity
Float value divide by zero
Infinity
Long value divide by zero
java.lang.ArithmeticException: / by zero
	at example.ArithmaticExceptionExample.main(ArithmaticExceptionExample.java:31)

How to fix Arithmetic Exception?

In above example:

Y/X or K/X result ArithmeticException

What happens here is that since both the dividend and the divisor are int, the operation is an integer division, whose result is rounded to an int. Remember that an int can only contain whole number (of limited range, some 4 billion numbers approximately) That’ s why throwing Arithmatic Exception. same case for long value.

Z/X or L/X result special value Infinity

Here float (L) or double (X) precision value  divide by zero store in float and double value that why result is special value Infinity.

 

[Solved] JAVA java.lang.StringIndexOutOfBoundsException: String index out of range: -1 Example

The java.lang.StringIndexOutOfBoundsException is RuntimeException or Checked Exception which is thrown by the methods of String class during index operations which indicate accessing index is either negative or greater than size of string itself.

java.lang.StringIndexOutOfBoundException is subclass of the IndexOutOfBoundException class which is used to indicate that accessing index to either an  Array, Vector or String, is out of Range.

String Class Methods where StringIndexOutOfBoundsException happen:

String object index range is between 0 to String.length() -1 if object access beyond this range lower or greater will throw StringIndexOutOfBoundsException .

  • public char charAt(int index);
  • public String substring(beginIndex);
  • public String subString(beginIndex,endIndex);
  • public CharSequence subSequence(beginIndex,endIndex);
  • public static String valueOf(char[] data, int offset, int count)

Example : Below example is for print all possible SubString for String FacingIssuesOnIT will throw StringIndexOutOfBoundException. on run time because index conditions are not properly handled.

package example;

public class StringIndexOutOfBounExceptionExample {

	public static void main(String[] args) {
		String str = "FacingIssueOnIT";
		System.out.println("*********All Substring of FacingIssuesonIT");
		for (int i = 0; i &lt;= str.length(); i++) {
			for (int j = i + 1; j &lt;= str.length(); j++) {
				System.out.println(str.substring(i, j - i));
			}
		}
	}
}

Output :

*********All Substring of FacingIssuesonIT
F
Fa
Fac
Faci
Facin
Facing
FacingI
FacingIs
FacingIss
FacingIssu
FacingIssue
FacingIssueO
FacingIssueOn
FacingIssueOnI

a
ac
aci
acin
acing
acingI
acingIs
acingIss
acingIssu
acingIssue
acingIssueO
acingIssueOn
Exception in thread "main" java.lang.StringIndexOutOfBoundsException: String index out of range: -1
	at java.lang.String.substring(Unknown Source)
	at example.StringIndexOutOfBounExceptionExample.main(StringIndexOutOfBounExceptionExample.java:10)

Issues with above Example :
Actually, your right edge of your substring function may be lower than the left one. For example, when i=(size-1) and j=size, you are going to compute substring(size-1, 1). This is the cause of you error.

Modified Example:

When you compare with above example loop condition are change SubString above range selection is modified.

package example;

public class StringIndexOutOfBounExceptionExample {

	public static void main(String[] args) {
		String str = "FacingIssueOnIT";
		System.out.println("*********All Substring of FacingIssuesOnIT");
		for (int i = 0; i &lt;=str.length()-1; i++) {
			for (int j = i+1 ; j &lt;= str.length(); j++) {
				//System.out.println(&quot;i=&quot;+i+&quot;,j=&quot;+ j);
				System.out.println(str.substring(i, j));
			}
		}
	}
}

Output:

*********All Substring of FacingIssuesOnIT
F
Fa
Fac
Faci
Facin
Facing
FacingI
FacingIs
FacingIss
FacingIssu
FacingIssue
FacingIssueO
FacingIssueOn
FacingIssueOnI
FacingIssueOnIT
a
ac
aci
acin
acing
acingI
acingIs
acingIss
acingIssu
acingIssue
acingIssueO
acingIssueOn
acingIssueOnI
acingIssueOnIT
c
ci
cin
cing
cingI
cingIs
cingIss
cingIssu
cingIssue
cingIssueO
cingIssueOn
cingIssueOnI
cingIssueOnIT
i
in
ing
ingI
ingIs
ingIss
ingIssu
ingIssue
ingIssueO
ingIssueOn
ingIssueOnI
ingIssueOnIT
n
ng
ngI
ngIs
ngIss
ngIssu
ngIssue
ngIssueO
ngIssueOn
ngIssueOnI
ngIssueOnIT
g
gI
gIs
gIss
gIssu
gIssue
gIssueO
gIssueOn
gIssueOnI
gIssueOnIT
I
Is
Iss
Issu
Issue
IssueO
IssueOn
IssueOnI
IssueOnIT
s
ss
ssu
ssue
ssueO
ssueOn
ssueOnI
ssueOnIT
s
su
sue
sueO
sueOn
sueOnI
sueOnIT
u
ue
ueO
ueOn
ueOnI
ueOnIT
e
eO
eOn
eOnI
eOnIT
O
On
OnI
OnIT
n
nI
nIT
I
IT
T

How to avoid StringIndexOutOfBoundException?

  • Debug program throughly after printing index values.
  • During operation check index value range between 0 to String.length()-1.

[Solved] NullPointerException In Java

NullPointerException is Runtime Exception and Unchecked Exception which thrown when trying to access an object which is having value as null.

What is Null Value?

Null value is a special  value which tell JVM this reference variable is assigned with no value as null. It means variable is not pointing to any object instances and not having any value.

Cases when NullPointerException Happen:

  • Accessing or modifying a null object’s field.
  • Taking length of null  from object like String, Array, ArrayList etc.
  • Invoking a method from a null object.
  • Calling method passing argument as wrapper object having null value while method having primitive arguments.
  • Accessing or modifying the slots of null object, as if it were an array.
  • Throwing null, as if it were a Throwable value.
  • When you try to synchronize over a null object.

Below are some examples for NullPointerException for above cases which showing how NullPointerException can happen in your code.

import java.util.List;

public class NullPointerExceptionExample {

	public static void main(String[] args) {
		//Example 1: String object assigned with null
		try
		{
		String str=null;
		if(str.length()&gt;0)//NullPointerException Here
			System.out.println("String Length :"+str.length());
		}
		catch(NullPointerException ex)
		{
			System.out.println("Example 1 :String object assigned with null");
			ex.printStackTrace();
		}

		//Example 2: Collection List Object Assigned with Null
		try
		{
		List list=null;
		if(list.size()&gt;0)//NullPointerException Here
			System.out.println("List Length :"+list.size());
		}
		catch(NullPointerException ex)
		{
			System.out.println("Example2 :Collection List object assigned with null");
			ex.printStackTrace();
		}

		//Example 3: Calling method with wrapper and method having primitive value
		try
		{
		Long acNumber=null;
		checkValue(acNumber);
		}
		catch(NullPointerException ex)
		{
			System.out.println("Example 3 :Calling method with wrapper and method having primitive type");
			ex.printStackTrace();
		}

		//Example 4 :Accessing slot of Array object having value as null
		try
		{
		Employee [] empArr=new Employee[10];
		System.out.println("Employee Name :"+empArr[5].getName());
		}
		catch(NullPointerException ex)
		{
			System.out.println("Example 4: Accessing slot of array objet having value as null");
			ex.printStackTrace();
		}
	}

	public static void checkValue(long acNumber)
	{
		System.out.println("Account Number:"+acNumber);
	}

	private class Employee
	{
		private int id;
		private String name;
		private double salary;

		public int getId() {
			return id;
		}
		public void setId(int id) {
			this.id = id;
		}
		public String getName() {
			return name;
		}
		public void setName(String name) {
			this.name = name;
		}
		public double getSalary() {
			return salary;
		}
		public void setSalary(double salary) {
			this.salary = salary;
		}

	}

}

Output :

Example 1 :String object assigned with null
java.lang.NullPointerException
	at example.NullPointerExceptionExample.main(NullPointerExceptionExample.java:12)
Example2 :Collection List object assigned with null
java.lang.NullPointerException
	at example.NullPointerExceptionExample.main(NullPointerExceptionExample.java:25)
Example 3 :Calling method with wrapper and method having primitive type
java.lang.NullPointerException
        at
example.NullPointerExceptionExample.main(NullPointerExceptionExample.java:38)
Example 4: Accessing slot of array objet having value as null
java.lang.NullPointerException
	at example.NullPointerExceptionExample.main(NullPointerExceptionExample.java:50)

 How to avoid NullPointerException?

NullPointerException can be avoid by using considering below points:

  • Need to ensure object as initialize properly before any operation check for null .
  • Need to ensure object reference is not null before calling any method or operation on fields.
  • While comparing string with literals. Instead of invoking the method from the null object, consider invoking it from the literal.
  • Before calling method check arguments if valid then only call methods.
  • Use of Ternary operator for initialize with some default value if object value is null.
  • We can use NullPointerException catch block when no action perform or if need to throw some Customize Exception or User Defined Exception.

In below example consider all above case for avoid NullPointerException.

public class NullPointerExceptionExample {

	public static void main(String[] args) {
		//Example 1: String object assigned with null
		//NullPointerException avoided by checking null before use
		String str=null;
		if(str!=null &amp;&amp; str.length()&gt;0)
			System.out.println("String Length :"+str.length());

		//Example 2: Object assigned with Null
		//NullPointerException avoided by using  null check before accessing field
		Employee emp=null;
		if(emp!=null)
			System.out.println("Employee Name :"+emp.getName());

		//Example 3: name is assigned with null
		//NullPointerException avoided by calling equals method from literals instead of name object.
		String name=null;
		if("FacingIssuesOnIT".equals(name))
		{
			System.out.println("Name is FacingIssuesOnIT");
		}
		else
		{
			System.out.println("Name is not FacingIssuesOnIT");
		}

		//Example 4: Calling method with wrapper as null and method having primitive value
		//NullPointerException avoided by checking null on method arguments before calling method.

		Long acNumber=null;
		if(acNumber!=null)
		{
		checkValue(acNumber);
		}

		//Example 5: Object reference value is null and need to perform operation
		//NullPointerException avoided by using ternary operator
		String siteName=null;
		name=siteName==null?"":siteName;
		System.out.println("Site Name Length:"+name.length() );

	}

	public static void checkValue(long acNumber)
	{
		System.out.println("Account Number:"+acNumber);
	}

	private class Employee
	{
		private int id;
		private String name;
		private double salary;

		public int getId() {
			return id;
		}
		public void setId(int id) {
			this.id = id;
		}
		public String getName() {
			return name;
		}
		public void setName(String name) {
			this.name = name;
		}
		public double getSalary() {
			return salary;
		}
		public void setSalary(double salary) {
			this.salary = salary;
		}

	}

}

Summary

  • Here you have learn about NullPointerException and Null value.
  • Cases with examples where NullPointerException can happen.
  • Al so explained with example to avoid NullPointerException by applying null check on object, compare String values by calling operations on literals and using ternary operators.

Armstrong Number JAVA Program

Armstrong number is a number which is equal to sum of digits raise to the power total number of digits in the number.

Example :

33+ 73 + 13 = 371

14 + 64 + 34 + 44=1634

Below is JAVA program to check Armstrong Number.

import java.util.Scanner;

class ArmstrongNumber {
	public static void main(String args[]) {
	int n, sum = 0, temp, remainder, digits = 0;

	Scanner in = new Scanner(System.in);
	System.out.println(&quot;Input a number to check if it is an Armstrong number&quot;);
	n = in.nextInt();

	temp = n;

	// Count number of digits

	while (temp != 0) {
		digits++;
		temp = temp / 10;
	}

	temp = n;

	while (temp != 0) {
		remainder = temp % 10;
		sum = sum + power(remainder, digits);
		temp = temp / 10;
	}

	if (n == sum)
		System.out.println(n + &quot; is an Armstrong number.&quot;);
	else
		System.out.println(n + &quot; is not an Armstrong number.&quot;);
	}

	static int power(int n, int r) {
		int c, p = 1;

		for (c = 1; c &lt;= r; c++)
			p = p * n;

		return p;
	}
}

More Info

For more Algorithms and Java Programing Test questions and sample code follow below links

How to Configure Filebeat, Kafka, Logstash Input , Elasticsearch Output and Kibana Dashboard

Filebeat, Kafka, Logstash, Elasticsearch and Kibana Integration is used for big organizations where applications deployed in production on hundreds/thousands of servers and scattered around different locations and need to do analysis on data from these servers on real time.

This integration helps mostly for log level analysis , tracking issues, anomalies with data and alerts on events of particular occurrence and where accountability measures.

By using these technology provide scalable architecture to enhance systems and decoupled of each other individually.

Why these Technology?

Filebeat :

  • Lightweight agent for shipping logs.
  • Forward and centralize files and logs.
  • Robust (Not miss a single beat)

Kafka:

  • Open source distributed, Steam Processing, Message Broker platform.
  • process stream data or transaction logs on real time.
  • fault-tolerant, high throughput, low latency platform for dealing real time data feeds.

Logstash:

  •  Open source, server-side data processing pipeline that accept data from a different  sources simultaneously.
  • Parse, Format, Transform data and send to different output sources.

Elasticsearch:

  • Elasticsearch is open source, distributed cross-platform.
  • Built on top of Lucene which provide full text search and provide NRT(Near real Time) search results.
  • Support RESTFUL search  by Elasticsearch REST

Kibana:

  • Open source
  • Provide window to view Elasticsearch data in form different charts and dashboard.
  • Provide way  searches and operation of data easily with respect to time interval.
  • Easily Imported by  any web application by embedded dashboards.

How Data flow works ?

In this integration filebeat will install in all servers where your application is deployed and filebeat will read and ship  latest logs changes from these servers to Kafka topic as configured for this application.

Logstash will subscribe log lines from kafka topic and perform parsing on these lines make relevant changes, formatting, exclude and include fields then send this processed data to Elasticsearch Indexes as centralize location from different servers.

Kibana  is linked with  Elasticsearch indexes which will help to do analysis by search, charts and dashboards .

FKLEK Integration

Design Architecture

In below configured architecture considering my application is deployed on three servers and each server having current log file name as App1.log . Our goal is read real time data from these servers and do analysis on these data.

FKLEK Arch Integration

Steps to Installation, Configuration and Start

Here first we will install Kafka and Elasticsearch run individually rest of tools will install and run sequence to test with data flow.  Initially install all in same machine  and test with sample data with below steps and at end of this post will tell about what changes need to make according to your servers.

  • Kafka Installation, Configuration and Start
  • Elasticsearch Installation,Configuration and Start
  • Filebeat Installation,Configuration and Start
  • Logstash Installation,Configuration and Start
  • Kibana Installation,Start and display.

Pre-Requisite

These Filebeat,Logstash, Elasticsearch and Kibana versions should be compatible better use latest from  https://www.elastic.co/downloads.

  • Java 8+
  • Linux Server
  • Filebeat 5.XX
  • Kafka 2.11.XX
  • Logstash 5.XX
  • Elasticsearch 5.XX
  • Kibana 5.XX

Note  : Make sure JDK 8 should be install  and JAVA_HOME environment variable point to JDK 8 home directory  wherever you want in install Elasticsearch, Logstash,Kibana and Kafka.

Window   : My computer ->right click-> Properties -> Advance System Settings->System Variable

Java_Home
Set JAVA_HOME

Linux : Go to your home directory/ sudo directory and below line as below .

export JAVA_HOME=/opt/app/facingissuesonit/jdk1.8.0_66

Sample Data

For testing we will use these sample log line which is having debug as well as stacktrace of logs and grok parsing of this example is designed according to it. For real time testing and actual data you can point to your server log files but you have to modify grok pattern in Logstash configuration accordingly.

2013-02-28 09:57:56,662 WARN  CreateSomethingActivationKey - WhateverException for User 49-123-345678 {{rid,US8cFAp5eZgAABwUItEAAAAI_dev01_443}{realsid,60A9772A136B9912B6FF0C3627A47090.dev1-a}}
2013-02-28 09:57:56,663 INFO  LMLogger - ERR1700 - u:null failures: 0  - Technical error {{rid,US8cFAp5eZgAABwUItEAAAAI_dev01_443}{realsid,60A9772A136B9912B6FF0C3627A47090.dev1-a}}
2013-02-28 09:57:56,668 ERROR SomeCallLogger - ESS10005 Cpc portalservices: Exception caught while writing log messege to MEA Call:  {}
java.sql.SQLSyntaxErrorException: ORA-00942: table or view does not exist

	at oracle.jdbc.driver.T4CTTIoer.processError(T4CTTIoer.java:445)
	at oracle.jdbc.driver.T4CTTIoer.processError(T4CTTIoer.java:396)
2013-02-28 10:04:35,723 INFO  EntryFilter - Fresh on request /portalservices/foobarwhatever {{rid,US8dogp5eZgAABwXPGEAAAAL_dev01_443}{realsid,56BA2AD41D9BB28AFCEEEFF927EE61C2.dev1-a}}

Create  App1.log file  in same machine where filebeat need to install and copy above logs lines in App1.log file.

Kafka Installation , Configuration and Start

Download latest version of Kafka from below link and use command to untar and installation in Linux server or if window just unzip downloaded file.

Download Link : https://kafka.apache.org/downloads

tar -zxvf kafka_2.11-0.10.0.0

For more configuration and start options follow Setup Kafka Cluster for Single Server/Broker

After download and untar/unzip file it will have below files and directory structure.

ls- l
drwxr-xr-x  3 facingissuesonit Saurabh   4096 Apr  3 05:18 bin
drwxr-xr-x  2 facingissuesonit Saurabh   4096 May  8 11:05 config
drwxr-xr-x 74 facingissuesonit Saurabh   4096 May 27 20:00 kafka-logs
drwxr-xr-x  2 facingissuesonit Saurabh   4096 Apr  3 05:17 libs
-rw-r--r--  1 facingissuesonit Saurabh  28824 Apr  3 05:17 LICENSE
drwxr-xr-x  2 facingissuesonit Saurabh 487424 May 27 20:00 logs
-rw-r--r--  1 facingissuesonit Saurabh    336 Apr  3 05:18 NOTICE
drwxr-xr-x  2 facingissuesonit Saurabh   4096 Apr  3 05:17 site-docs

For more details about all these files,configuration option and other integration options follow Kafka Tutorial.

Make below changes in files config/zookeeper.properties and config/server.properties

config/zookeeper.properties

clientPort=2181
config/server.properties:

broker.id=0
listeners=PLAINTEXT://:9092
log.dir=/kafka-logs
zookeeper.connect=localhost:2181

Now Kafka is configured and ready to run. Use below command to start zookeeper and Kafka server as  background process.

screen -d -m bin/zookeeper-server-start.sh config/zookeeper.properties
screen -d -m bin/kafka-server-start.sh config/server.properties

To test  Kafka  install successfully you can check by running Kafka process on Linux “ps -ef|grep kafka” or steps for consumer and producer to/from topic in Setup Kafka Cluster for Single Server/Broker.

Elasticsearch Installation,Configuration and Start

Download latest version of Elasticsearch from below link and use command to untar and installation in Linux server or if window just unzip downloaded file.

Download Link : https://www.elastic.co/downloads/elasticsearch

tar -zxvf elasticsearch-5.4.0.tar.gz

It will show below files and directory structure for Elasticsearch.

drwxr-xr-x  2 facingissuesonit Saurabh   4096 Apr 25 19:20 bin
drwxr-xr-x  3 facingissuesonit Saurabh   4096 May 13 17:27 config
drwxr-xr-x  3 facingissuesonit Saurabh   4096 Apr 24 15:56 data
drwxr-xr-x  2 facingissuesonit Saurabh   4096 Apr 17 10:55 lib
-rw-r--r--  1 facingissuesonit Saurabh  11358 Apr 17 10:50 LICENSE.txt
drwxr-xr-x  2 facingissuesonit Saurabh   4096 May 28 05:00 logs
drwxr-xr-x 12 facingissuesonit Saurabh   4096 Apr 17 10:55 modules
-rw-r--r--  1 facingissuesonit Saurabh 194187 Apr 17 10:55 NOTICE.txt
drwxr-xr-x  2 facingissuesonit Saurabh   4096 Apr 17 10:55 plugins
-rw-r--r--  1 facingissuesonit Saurabh   9540 Apr 17 10:50 README.textile

Before going to start Elasticsearch need to make some basic changes in config/elasticsearch.yml file for cluster  and node name. You can configure it based on you application or organization name.

cluster.name: FACING-ISSUE-IN-IT
node.name: TEST-NODE-1
#network.host: 0.0.0.0
http.port: 9200

Now we are ready with elasticsearch configuration and time start elasticsearch. We can use below command to run elasticsearch in background.

screen -d -m  /bin/elasticsearch

For  checking elasticsearch starts successfully you can use below url on browser  to know cluster status . You will get result like below.

http://localhost:9200/_cluster/health?pretty

or as below if network.host configured

http://elasticseverIp:9200/_cluster/health?pretty

Result :

{
  "cluster_name" : "FACING-ISSUE-IN-IT",
  "status" : "green",
  "timed_out" : false,
  "number_of_nodes" : 1,
  "number_of_data_nodes" : 1,
  "active_primary_shards" : 0,
  "active_shards" : 0,
  "relocating_shards" : 0,
  "initializing_shards" : 0,
  "unassigned_shards" : 0,
  "delayed_unassigned_shards" : 0,
  "number_of_pending_tasks" : 0,
  "number_of_in_flight_fetch" : 0,
  "task_max_waiting_in_queue_millis" : 0,
  "active_shards_percent_as_number" : 100.0
}

Filebeat Installation, Configuration and Start

Download latest version of filebeat from  below link and use  command to untar  and installation in Linux server. or if window just unzip downloaded file.

Download Link : https://www.elastic.co/downloads/beats/filebeat

tar -zxvf filebeat-&lt;version&gt;.tar.gz

For more configuration and start options follow Filebeat Download,Installation and Start/Run

After download and untar/unzip file it will have below files and directory structure.

ls- l
-rwxr-xr-x 1 facingissuesonit Saurabh 14908742 Jan 11 14:11 filebeat
-rw-r--r-- 1 facingissuesonit Saurabh    31964 Jan 11 14:11 filebeat.full.yml
-rw-r--r-- 1 facingissuesonit Saurabh     3040 Jan 11 14:11 filebeat.template-es2x.json
-rw-r--r-- 1 facingissuesonit Saurabh     2397 Jan 11 14:11 filebeat.template.json
-rw-r--r-- 1 facingissuesonit Saurabh     4196 Jan 11 14:11 filebeat.yml
-rw-r--r-- 1 facingissuesonit Saurabh      811 Jan 11 14:10 README.md
drwxr-xr-x 2 facingissuesonit Saurabh     4096 Jan 11 14:11 scripts

For more details about all these files,configuration option and other integration options follow Filebeat Tutorial.

Now filebeat is installaed and need to make below changes in filebeat.full.yml file

  • Inside prospectors section change paths to your log file location as
paths:
-/opt/app/facingissuesonit/App1.log
  • Comment out Elasticsearch Output default properties as below
#output.elasticsearch:
#hosts: [&quot;localhost:9200&quot;]
  • Configure multiline option as below so that all stacktrace line which are not starting with date  can we consider as single line.
multiline.pattern: ^\d
multiline.negate: true
multiline.match: after

For learn more on filebeat multiline configuration follow Filebeat Multiline Configuration Changes for Object, StackTrace and XML

  • Inside Kafka Output section update these properties hosts and topic. if Kafka on same machine then use localhost else update with IP of kafka machine.
output.kafka:
 hosts: [&quot;localhost:9092&quot;]
 topic: APP-1-TOPIC

For more on Logging configuration follow link Filebeat, Logging Configuration.

Now filebeat is configured and ready to start with  below command, it will read from configured prospector for file App1.log continiously and publish log line events to Kafka . It will also create topic as APP-1-TOPIC in Kafka if not exist.

./filebeat -e -c filebeat.full.yml -d &quot;publish&quot;

On console it will display output as below for sample lines.

2017/05/28 00:24:27.991828 client.go:184: DBG  Publish: {
  &quot;@timestamp&quot;: &quot;2017-05-28T00:24:22.991Z&quot;,
  &quot;beat&quot;: {
    &quot;hostname&quot;: &quot;sg02870&quot;,
    &quot;name&quot;: &quot;sg02870&quot;,
    &quot;version&quot;: &quot;5.1.2&quot;
  },
  &quot;input_type&quot;: &quot;log&quot;,
  &quot;message&quot;: &quot;2013-02-28 09:57:56,662 WARN  CreateSomethingActivationKey - WhateverException for User 49-123-345678 {{rid,US8cFAp5eZgAABwUItEAAAAI_dev01_443}{realsid,60A9772A136B9912B6FF0C3627A47090.dev1-a}}&quot;,
  &quot;offset&quot;: 194,
  &quot;source&quot;: &quot;/opt/app/facingissuesonit/App1.log&quot;,
  &quot;type&quot;: &quot;log&quot;
}
2017/05/28 00:24:27.991907 client.go:184: DBG  Publish: {
  &quot;@timestamp&quot;: &quot;2017-05-28T00:24:22.991Z&quot;,
  &quot;beat&quot;: {
    &quot;hostname&quot;: &quot;sg02870&quot;,
    &quot;name&quot;: &quot;sg02870&quot;,
    &quot;version&quot;: &quot;5.1.2&quot;
  },
  &quot;input_type&quot;: &quot;log&quot;,
  &quot;message&quot;: &quot;2013-02-28 09:57:56,663 INFO  LMLogger - ERR1700 - u:null failures: 0  - Technical error {{rid,US8cFAp5eZgAABwUItEAAAAI_dev01_443}{realsid,60A9772A136B9912B6FF0C3627A47090.dev1-a}}&quot;,
  &quot;offset&quot;: 375,
  &quot;source&quot;: &quot;/opt/app/facingissuesonit/App1.log&quot;,
  &quot;type&quot;: &quot;log&quot;
}
2017/05/28 00:24:27.991984 client.go:184: DBG  Publish: {
  &quot;@timestamp&quot;: &quot;2017-05-28T00:24:22.991Z&quot;,
  &quot;beat&quot;: {
    &quot;hostname&quot;: &quot;sg02870&quot;,
    &quot;name&quot;: &quot;sg02870&quot;,
    &quot;version&quot;: &quot;5.1.2&quot;
  },
  &quot;input_type&quot;: &quot;log&quot;,
  &quot;message&quot;: &quot;2013-02-28 09:57:56,668 ERROR SomeCallLogger - ESS10005 Cpc portalservices: Exception caught while writing log messege to MEA Call:  {}\njava.sql.SQLSyntaxErrorException: ORA-00942: table or view does not exist\n\n\tat oracle.jdbc.driver.T4CTTIoer.processError(T4CTTIoer.java:445)\n\tat oracle.jdbc.driver.T4CTTIoer.processError(T4CTTIoer.java:396)&quot;,
  &quot;offset&quot;: 718,
  &quot;source&quot;: &quot;/opt/app/facingissuesonit/App1.log&quot;,
  &quot;type&quot;: &quot;log&quot;
}
2017/05/28 00:24:27.991984 client.go:184: DBG  Publish: {
  &quot;@timestamp&quot;: &quot;2017-05-28T00:24:22.992Z&quot;,
  &quot;beat&quot;: {
    &quot;hostname&quot;: &quot;sg02870&quot;,
    &quot;name&quot;: &quot;sg02870&quot;,
    &quot;version&quot;: &quot;5.1.2&quot;
  },
  &quot;input_type&quot;: &quot;log&quot;,
  &quot;message&quot;: &quot;2013-02-28 10:04:35,723 INFO  EntryFilter - Fresh on request /portalservices/foobarwhatever {{rid,US8dogp5eZgAABwXPGEAAAAL_dev01_443}{realsid,56BA2AD41D9BB28AFCEEEFF927EE61C2.dev1-a}}&quot;,
  &quot;offset&quot;: 902,
  &quot;source&quot;: &quot;/opt/app/facingissuesonit/App1.log&quot;,
  &quot;type&quot;: &quot;log&quot;
}

Now you can see from above filebeat debug statements publish event 3 is having multiline statements with stacktrace exception and each debug will have these fields like.

@timestamp:  Timestamp of data shipped.

beat.hostname : filebeat machine name from where data is shipping.

beat.version: which version of filebeat installed on server that help for compatibility check on target end.

message : Log line from logs file or multline log lines

offset: it’s represent inode value in source file

source :  it’s file name from where logs were read

Now time to check data is publish to Kafka topic or not. For this go to below directory  and you will see two files as xyz.index and xyz.log for maintaining data offset and messages.

{Kafka_home}/kafk_logs/APP-1-TOPIC
          00000000000000000000.log
          00000000000000000000.index

Now your server log lines are in Kafka topic for reading and parsing  by Logstash and send it to elasticsearch for doing analysis/search on this data.

Logstash Installation, Configuration and Start

Download latest version of Logstash from below link and use command to untar and installation in Linux server or if window just unzip downloaded file.

Download Link : https://www.elastic.co/downloads/logstash

tar -zxvf logstash-5.4.0.tar.gz

It will show below file and directory structure.

drwxr-xr-x 2 facingissuesonit Saurabh   4096 Apr 20 11:27 bin
-rw-r--r-- 1 facingissuesonit Saurabh 111569 Mar 22 23:49 CHANGELOG.md
drwxr-xr-x 2 facingissuesonit Saurabh   4096 Apr 20 11:27 config
-rw-r--r-- 1 facingissuesonit Saurabh   2249 Mar 22 23:49 CONTRIBUTORS
drwxr-xr-x 3 facingissuesonit Saurabh   4096 Apr 20 12:07 data
-rw-r--r-- 1 facingissuesonit Saurabh   3945 Mar 22 23:55 Gemfile
-rw-r--r-- 1 facingissuesonit Saurabh  21544 Mar 22 23:49 Gemfile.jruby-1.9.lock
drwxr-xr-x 5 facingissuesonit Saurabh   4096 Apr 20 11:27 lib
-rw-r--r-- 1 facingissuesonit Saurabh    589 Mar 22 23:49 LICENSE
drwxr-xr-x 2 facingissuesonit Saurabh   4096 May 21 00:00 logs
drwxr-xr-x 4 facingissuesonit Saurabh   4096 Apr 20 11:27 logstash-core
drwxr-xr-x 3 facingissuesonit Saurabh   4096 Apr 20 11:27 logstash-core-event-java
drwxr-xr-x 3 facingissuesonit Saurabh   4096 Apr 20 11:27 logstash-core-plugin-api
drwxr-xr-x 3 facingissuesonit Saurabh   4096 Apr 20 11:27 logstash-core-queue-jruby
-rw-r--r-- 1 facingissuesonit Saurabh  28114 Mar 22 23:56 NOTICE.TXT
drwxr-xr-x 4 facingissuesonit Saurabh   4096 Apr 20 11:27 vendor

Before going to start Logstash need to create configuration file for taking input data from Kafka and parse these data in respected fields and send it elasticsearch. Create file logstash-app1.conf in logstash bin directory with below content.

/bin/logstash-app1.conf

input {
     kafka {
            bootstrap_servers =&gt; 'localhost:9092'
            topics =&gt; [&quot;APP-1-TOPIC&quot;]
            codec =&gt; json {}
          }
}
filter
{
//parse log line
      grok
	{
	match =&gt; {&quot;message&quot; =&gt; &quot;\A%{TIMESTAMP_ISO8601:timestamp}\s+%{LOGLEVEL:loglevel}\s+(?&lt;logger&gt;(?:[a-zA-Z0-9-]+\.)*[A-Za-z0-9$]+)\s+(-\s+)?(?=(?&lt;msgnr&gt;[A-Z]+[0-9]{4,5}))*%{DATA:message}({({[^}]+},?\s*)*})?\s*$(?&lt;stacktrace&gt;(?m:.*))?&quot; }
	}  

    #Remove unused fields
    #mutate { remove_field =&gt;[&quot;beat&quot;,&quot;@version&quot; ]}
}
output {
    #Output result sent to elasticsearch and dynamically create array
    elasticsearch {
        index  =&gt; &quot;app1-logs-%{+YYYY.MM.dd}&quot;
        hosts =&gt; [&quot;localhost:9200&quot;]
        sniffing =&gt; false
  	}

     #Sysout logs
     stdout
       {
         codec =&gt; rubydebug
       }
}

To test your configuration file you can use below command.


./logstash -t -f logstash-app1.conf

If  we get result OK from above command run below to start reading and parsing data from Kafka topic.


./logstash -f logstash-app1.conf

For design your own grok pattern for you logs line formatting you can follow below link that will help to generate incrementally and also provide some sample logs grok.

http://grokdebug.herokuapp.com and http://grokconstructor.appspot.com/

Logstash console will show parse data as below  and you can remove unsed fields for storing in elasticsearch by uncomment mutate section from configuration file.

{
    &quot;@timestamp&quot; =&gt; 2017-05-28T23:47:42.160Z,
        &quot;offset&quot; =&gt; 194,
      &quot;loglevel&quot; =&gt; &quot;WARN&quot;,
        &quot;logger&quot; =&gt; &quot;CreateSomethingActivationKey&quot;,
          &quot;beat&quot; =&gt; {
        &quot;hostname&quot; =&gt; &quot;zlp0287k&quot;,
            &quot;name&quot; =&gt; &quot;zlp0287k&quot;,
         &quot;version&quot; =&gt; &quot;5.1.2&quot;
    },
    &quot;input_type&quot; =&gt; &quot;log&quot;,
      &quot;@version&quot; =&gt; &quot;1&quot;,
        &quot;source&quot; =&gt; &quot;/opt/app/facingissuesonit/App1.log&quot;,
       &quot;message&quot; =&gt; [
        [0] &quot;2013-02-28 09:57:56,662 WARN  CreateSomethingActivationKey - WhateverException for User 49-123-345678 {{rid,US8cFAp5eZgAABwUItEAAAAI_dev01_443}{realsid,60A9772A136B9912B6FF0C3627A47090.dev1-a}}&quot;,
        [1] &quot;WhateverException for User 49-123-345678 &quot;
    ],
          &quot;type&quot; =&gt; &quot;log&quot;,
     &quot;timestamp&quot; =&gt; &quot;2013-02-28 09:57:56,662&quot;
}
{
         &quot;msgnr&quot; =&gt; &quot;ERR1700&quot;,
    &quot;@timestamp&quot; =&gt; 2017-05-28T23:47:42.160Z,
        &quot;offset&quot; =&gt; 375,
      &quot;loglevel&quot; =&gt; &quot;INFO&quot;,
        &quot;logger&quot; =&gt; &quot;LMLogger&quot;,
          &quot;beat&quot; =&gt; {
        &quot;hostname&quot; =&gt; &quot;zlp0287k&quot;,
            &quot;name&quot; =&gt; &quot;zlp0287k&quot;,
         &quot;version&quot; =&gt; &quot;5.1.2&quot;
    },
    &quot;input_type&quot; =&gt; &quot;log&quot;,
      &quot;@version&quot; =&gt; &quot;1&quot;,
        &quot;source&quot; =&gt; &quot;/opt/app/facingissuesonit/App1.log&quot;,
       &quot;message&quot; =&gt; [
        [0] &quot;2013-02-28 09:57:56,663 INFO  LMLogger - ERR1700 - u:null failures: 0  - Technical error {{rid,US8cFAp5eZgAABwUItEAAAAI_dev01_443}{realsid,60A9772A136B9912B6FF0C3627A47090.dev1-a}}&quot;,
        [1] &quot;ERR1700 - u:null failures: 0  - Technical error &quot;
    ],
          &quot;type&quot; =&gt; &quot;log&quot;,
     &quot;timestamp&quot; =&gt; &quot;2013-02-28 09:57:56,663&quot;
}
{
        &quot;offset&quot; =&gt; 718,
        &quot;logger&quot; =&gt; &quot;SomeCallLogger&quot;,
    &quot;input_type&quot; =&gt; &quot;log&quot;,

       &quot;message&quot; =&gt; [
        [0] &quot;2013-02-28 09:57:56,668 ERROR SomeCallLogger - ESS10005 Cpc portalservices: Exception caught while writing log messege to MEA Call:  {}\njava.sql.SQLSyntaxErrorException: ORA-00942: table or view does not exist\n\n\tat oracle.jdbc.driver.T4CTTIoer.processError(T4CTTIoer.java:445)\n\tat oracle.jdbc.driver.T4CTTIoer.processError(T4CTTIoer.java:396)&quot;,
        [1] &quot;ESS10005 Cpc portalservices: Exception caught while writing log messege to MEA Call:  &quot;
    ],
          &quot;type&quot; =&gt; &quot;log&quot;,
         &quot;msgnr&quot; =&gt; &quot;ESS10005&quot;,
    &quot;@timestamp&quot; =&gt; 2017-05-28T23:47:42.160Z,
    &quot;stacktrace&quot; =&gt; &quot;\njava.sql.SQLSyntaxErrorException: ORA-00942: table or view does not exist\n\n\tat oracle.jdbc.driver.T4CTTIoer.processError(T4CTTIoer.java:445)\n\tat oracle.jdbc.driver.T4CTTIoer.processError(T4CTTIoer.java:396)&quot;,
      &quot;loglevel&quot; =&gt; &quot;ERROR&quot;,
          &quot;beat&quot; =&gt; {
        &quot;hostname&quot; =&gt; &quot;zlp0287k&quot;,
            &quot;name&quot; =&gt; &quot;zlp0287k&quot;,
         &quot;version&quot; =&gt; &quot;5.1.2&quot;
    },
      &quot;@version&quot; =&gt; &quot;1&quot;,
     &quot;timestamp&quot; =&gt; &quot;2013-02-28 09:57:56,668&quot;
}
{
    &quot;@timestamp&quot; =&gt; 2017-05-28T23:47:42.160Z,
        &quot;offset&quot; =&gt; 903,
      &quot;loglevel&quot; =&gt; &quot;INFO&quot;,
        &quot;logger&quot; =&gt; &quot;EntryFilter&quot;,
          &quot;beat&quot; =&gt; {
        &quot;hostname&quot; =&gt; &quot;zlp0287k&quot;,
            &quot;name&quot; =&gt; &quot;zlp0287k&quot;,
         &quot;version&quot; =&gt; &quot;5.1.2&quot;
    },
    &quot;input_type&quot; =&gt; &quot;log&quot;,
      &quot;@version&quot; =&gt; &quot;1&quot;,

       &quot;message&quot; =&gt; [
        [0] &quot;2013-02-28 10:04:35,723 INFO  EntryFilter - Fresh on request /portalservices/foobarwhatever {{rid,US8dogp5eZgAABwXPGEAAAAL_dev01_443}{realsid,56BA2AD41D9BB28AFCEEEFF927EE61C2.dev1-a}}\n&quot;,
        [1] &quot;Fresh on request /portalservices/foobarwhatever &quot;
    ],
          &quot;type&quot; =&gt; &quot;log&quot;,
     &quot;timestamp&quot; =&gt; &quot;2013-02-28 10:04:35,723&quot;
}

To test on elasticsearch end your data sent  successfully  you can use this url
http://localhost:9200/_cat/indices  on your browser and will display created index with current date.

yellow open app1-logs-2017.05.28                             Qjs6XWiFQw2zsiVs9Ks6sw 5 1         4     0  47.3kb  47.3kb

Kibana Installation, Configuration and Start

Download latest version of Kibana from below link and use command to untar and installation in Linux server or if window just unzip downloaded file.

Download Link : https://www.elastic.co/downloads/kibana

tar -zxvf kibana-5.4.0.tar.gz

It will show below files and directory structure for kibana.

ls -l
drwxr-xr-x   2 facingissuesonit Saurabh   4096 May 22 14:23 bin
drwxr-xr-x   2 facingissuesonit Saurabh   4096 Apr 25 18:58 config
drwxr-xr-x   2 facingissuesonit Saurabh   4096 Apr 25 11:54 data
-rw-r--r--   1 facingissuesonit Saurabh    562 Apr 17 12:04 LICENSE.txt
drwxr-xr-x   6 facingissuesonit Saurabh   4096 Apr 17 12:04 node
drwxr-xr-x 485 facingissuesonit Saurabh  20480 Apr 17 12:04 node_modules
-rw-r--r--   1 facingissuesonit Saurabh 660429 Apr 17 12:04 NOTICE.txt
drwxr-xr-x   3 facingissuesonit Saurabh   4096 Apr 17 12:04 optimize
-rw-r--r--   1 facingissuesonit Saurabh    702 Apr 17 12:04 package.json
drwxr-xr-x   2 facingissuesonit Saurabh   4096 May 22 12:29 plugins
-rw-r--r--   1 facingissuesonit Saurabh   4909 Apr 17 12:04 README.txt
drwxr-xr-x  10 facingissuesonit Saurabh   4096 Apr 17 12:04 src
drwxr-xr-x   3 facingissuesonit Saurabh   4096 Apr 17 12:04 ui_framework
drwxr-xr-x   2 facingissuesonit Saurabh   4096 Apr 17 12:04 webpackShims

Before going to start Kibana need to make some basic changes in config/kibana.yml file make below changes after uncomment these properties file.

server.port: 5601
server.host: localhost
elasticsearch.url: "http://localhost:9200"

Now we are ready with Kibana configuration and time start Kibana. We can use below command to run Kibana in background.

screen -d -m  /bin/kibana

Kibana take time to start and we can test it by using below url in browser

http://localhost:5601/

For checking this data  in Kibana open above url in browser go to management tab on left side menu -> Index Pattern -> Click on Add New

Enter Index name or pattern and time field name as in below screen  and click on create button.

Kibana index setting
Index Pattern Settings

Now go to Discover Tab and select index as app1-log* will display data as below.

kibana discover data

Now make below changes according to  your application specification .

Filebeat :

  • update prospector path to your log directory current file
  •  Move Kafka on different machine because Kafka will single location where receive shipped data from different servers. Update localhost with same IP of kafka server in Kafka output section of filebeat.full.yml file  for hosts properties.
  • Copy same filebeat setup on all servers from where you application deployed and need to read logs.
  • Start all filebeat instances on each Server.

Elasticsearch :

  • Uncomment network.host properties from elasticsearch.yml file for accessing by  IP address.

Logstash:

  • Update localhost in logstash-app1.conf file input section with Kafka machine IP.
  • change grok pattern in filter section according to your logs format. You can take help from below url for incrementally design. http://grokdebug.herokuapp.com and http://grokconstructor.appspot.com/
  • Update localhost output section for elasticsearch with IP if moving on different machine.

Kibana:

  • update localhost in kibana.yml file for elasticsearch.url properties with IP if kibana on different machine.

Conclusion :

In this tutorial considers below points :

  • Installation of Filebeat, Kafka, Logstash, Elasticsearch and Kibana.
  • Filebeat is configured to shipped logs to Kafka Message Broker.
  • Logstash configured to read logs line from Kafka topic , Parse and shipped to Elasticsearch.
  • Kibana show these Elasticsearch information in form of chart and dashboard to users for doing analysis.

Read More

To read more on Filebeat, Kafka, Elasticsearch  configurations follow the links and Logstash Configuration,Input Plugins, Filter Plugins, Output Plugins, Logstash Customization and related issues follow Logstash Tutorial and Logstash Issues.

Hope this blog was helpful for you.

Leave you feedback to enhance more on this topic so that make it more helpful for others.

Reference  :

 https://www.elastic.co/products

 

Elasticsearch REST Index Manager Auto Client for CRUD

Elasticsearch 5 REST Java Index Manager Auto Client can  help to manage index life from client end by setting configuration for keeping  index   open, close, delete indexes  for this no any third party tool required.

Below steps for auto  index management will save your time of index management manually and will take care of index life based on configure time.

Pre-requisite

  • Minimum requirement for Java 8 version required.
  • Add dependency for Elasticsearch REST and JSON Mapping in your pom.xml or add in your class path.
  • Index name format should be like IndexName-2017.06.10 for Ex. app1-logs-2017.06.08 if you have different date format change accordingly in below code.

We will follow below steps to create this client and auto run:

  • Create Java Maven Project ElasticsearchAutoIndexManager.
  • Add ElasticSearchIndexManagerClient in Project.
  • Test
  • Create auto run jar for project
  • Create script file for run auto jar
  • Create Cron Tab configuration for schedule and receive email alert.

Create Java Maven Project ElasticsearchAutoIndexManager

Create console based JAVA maven project as in below screen shot with name as ElasticsearchAutoIndexManager . To know more about Console based Java maven project follow link How to create Maven Java Console Project?

Elasticsearch REST Auto Client

Add below dependency in pom.xml file

&lt;!--Elasticsearch REST jar--&gt;
&lt;dependency&gt;
			&lt;groupId&gt;org.elasticsearch.client&lt;/groupId&gt;
			&lt;artifactId&gt;rest&lt;/artifactId&gt;
			&lt;version&gt;5.1.2&lt;/version&gt;
&lt;/dependency&gt;
&lt;!--Jackson jar for mapping json to Java --&gt;
&lt;dependency&gt;
			&lt;groupId&gt;com.fasterxml.jackson.core&lt;/groupId&gt;
			&lt;artifactId&gt;jackson-databind&lt;/artifactId&gt;
			&lt;version&gt;2.8.5&lt;/version&gt;
&lt;/dependency&gt;

Add below ElasticSearchIndexManagerClient class in com.facingissuesonit.es package and make below constant fields changes as per your server info and requirement.

Set INDEX_NO_ACTION_TIME so that till these days difference no action will take. For Example as set 2 means till  two days index will searchable in system.

private static final int INDEX_NO_ACTION_TIME = 2; 

Set INDEX_CLOSE_TIME so that all indexes will in close status means exist in elasticsearch server but not searchable.For Example as set 5 means if index life is more than five days  will close these indexes and keep it as long as Index delete time not reach.

private static final int INDEX_CLOSE_TIME = 5; 

Set INDEX_DELETE_TIME decide when to delete these indexes which match this criteria. For example as set 15 means will delete all indexes which are having index life more than 15 days.

private static final int INDEX_DELETE_TIME = 15;

private static final String ELASTICSEARCH_SERVER = “ServerHost”;

private static final int ELASTICSEARCH_SERVER_PORT = 9200;

Note : Set proxy server and login credential information if required else comment out.

package com.facingissuesonit.es;

import java.io.IOException;
import java.io.InputStream;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;

import com.fasterxml.jackson.databind.ObjectMapper;

public class ElasticSearchIndexManagerClient {
	private static final int INDEX_NO_ACTION_TIME = 2;
	private static final int INDEX_CLOSE_TIME = 5;
	private static final int INDEX_DELETE_TIME = 15;
	private static final String ELASTICSEARCH_SERVER = &quot;ServerHost&quot;;
	private static final int ELASTICSEARCH_SERVER_PORT = 9200;
	public static void main(String[] args) {
		RestClient client;
		String indexName = &quot;&quot;, indexDateStr = &quot;&quot;;
		LocalDate indexDate = null;
		long days = 0;
		final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(&quot;yyyy-MM-dd&quot;);
		final LocalDate todayLocalDate = LocalDate.now();

		try {
			ElasticSearchIndexManagerClient esManager=new ElasticSearchIndexManagerClient();
			//Get Connection from Elasticsearch
			client=esManager.getElasticsearchConnectionClient();
			if(client!=null)
			{
				IndexDetail[] indexList = esManager.getIndexDetailList(client);

				if (indexList != null &amp;&amp; indexList.length &gt; 0) {
					for (IndexDetail indexDetail : indexList) {
						indexName = indexDetail.getIndexName();
						System.out.println(indexName);
						indexDateStr = indexName.substring(indexName.lastIndexOf(&quot;-&quot;) + 1);
						//Below code is for getting number of days difference from index creation ad current date
						try {
							indexDate = LocalDate.parse(indexDateStr.replace('.', '-'), formatter);
							days = ChronoUnit.DAYS.between(indexDate, todayLocalDate);
							esManager.performAction(indexDetail, days,client);
						} catch (Exception ex) {
							System.out.println(&quot;Index is not having formatted date as required : yyyy.MM.dd :&quot;+indexName);
						}
					}
				}
			}
		} catch (Exception ex) {
			System.out.println(&quot;Exception found while index management&quot;);
			ex.printStackTrace();
			System.exit(1);
		} finally {
			System.out.println(&quot;Index Management successfully completed&quot;);
			System.exit(0);
		}
	}
	//Get Elasticsearch Connection
	private RestClient getElasticsearchConnectionClient() {
		final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
		credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(&quot;userid&quot;, &quot;password&quot;));

		RestClient client = RestClient
				.builder(new HttpHost(ELASTICSEARCH_SERVER,ELASTICSEARCH_SERVER_PORT))
				.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {

					public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
						return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)
								.setProxy(new HttpHost(&quot;ProxyHost&quot;, &quot;ProxyPort&quot;));

					}
				}).setMaxRetryTimeoutMillis(60000).build();
		return client;
	}
	//Get List of Indexes in Elaticsearxh Server
	public IndexDetail[] getIndexDetailList(RestClient client)
	{
		IndexDetail[] indexDetails=null;
		HttpEntity in=null;
		try
		{
		ObjectMapper jacksonObjectMapper = new ObjectMapper();
		Response response = client.performRequest(&quot;GET&quot;, &quot;/_cat/indices?format=json&amp;pretty&quot;, Collections.singletonMap(&quot;pretty&quot;, &quot;true&quot;));
		in =response.getEntity();
		indexDetails=jacksonObjectMapper.readValue(in.getContent(), IndexDetail[].class);
		System.out.println(&quot;Index found :&quot;+indexDetails.length);
		}
		catch(IOException ex)
		{
			ex.printStackTrace();
		}

		return indexDetails;
	}
	//This Method Decide what action need to take based based Index creation date and configured date for No Action, close and Delete indexes
	private  void performAction(IndexDetail indexDetail, long days,RestClient client) {
		String indexName = indexDetail.getIndexName();
		if (days &gt;= INDEX_NO_ACTION_TIME) {
			if (!(indexDetail.getStatus() != null &amp;&amp; indexDetail.getStatus().equalsIgnoreCase(&quot;close&quot;))) {
				// Close index condition
				if (days &gt;= INDEX_CLOSE_TIME) {
					System.out.println(&quot;Close Index :&quot; + indexName);
					closeIndex(indexName,client);
				}
			}
			// Delete index condition
			if (days &gt;= INDEX_DELETE_TIME) {
				if (!(indexDetail.getStatus() != null &amp;&amp; indexDetail.getStatus().equalsIgnoreCase(&quot;close&quot;))) {
					System.out.println(&quot;Delete Index :&quot; + indexName);
					deleteIndex(indexName,client);
				} else {
					System.out.println(&quot;Delete Close Index :&quot; + indexName);
					deleteCloseIndex(indexName,client);
				}
			}
		}
	}

	// Operation on Indexes
		private  void closeIndex(String indexName,RestClient client) {

			flushIndex(indexName,client);
			postDocuments(indexName + &quot;/_close&quot;, client);
			System.out.println(&quot;Close Index :&quot; + indexName);
		}

		private  void deleteIndex(String indexName,RestClient client) {
			flushIndex(indexName,client);
			deleteDocument(indexName,client);
			System.out.println(&quot;Delete Index :&quot; + indexName);
		}

		private  void deleteCloseIndex(String indexName,RestClient client) {
			openIndex(indexName,client);
			flushIndex(indexName,client);
			deleteDocument(indexName, client);
			System.out.println(&quot;Delete Close Index :&quot; + indexName);
		}

		private  void openIndex(String indexName,RestClient client) {
			postDocuments(indexName + &quot;/_open&quot;, client);
			System.out.println(&quot;Open Index :&quot; + indexName);
		}

		private  void flushIndex(String indexName,RestClient client) {
			postDocuments(indexName + &quot;/_flush&quot;, client);
			System.out.println(&quot;Flush Index :&quot; + indexName);
			try {
				Thread.sleep(3000);
			} catch (InterruptedException ex) {
				ex.printStackTrace();
			}
		}
		//POST perform action used for creation and updation indexes
		public InputStream postDocuments(String endPoint,RestClient client)
		{
			InputStream in=null;
			Response response=null;
			try
			{
				response = client.performRequest(&quot;POST&quot;, endPoint, Collections.&lt;String, String&gt;emptyMap());
				in =response.getEntity().getContent();
			}
			catch(IOException ex)
			{
				System.out.println(&quot;Exception in post Documents :&quot;);
				ex.printStackTrace();
			}
			return in;
		}
		//DELETE perform action use for Deletion of Index
		public InputStream deleteDocument(String endPoint,RestClient client)
		{
			InputStream in=null;
			try
			{

		    Response response = client.performRequest(&quot;DELETE&quot;, endPoint, Collections.singletonMap(&quot;pretty&quot;, &quot;true&quot;));
			in =response.getEntity().getContent();
			}
			catch(IOException ex)
			{
				System.out.println(&quot;Exception in delete Documents :&quot;);
				ex.printStackTrace();
			}
			return in;
		}

}

In above code pretty state forward and step by step. Let’s me explain about below operation.

Open :  Index status open keep index available for searching and we can perform below operation like close and delete on indexes when it open status. For making Index open we can use below command in curl .

curl POST /indexName-2017.06.10/_open

Flush: This operation  is required before executing close and delete operation on indexes so that all running transactions on indexes complete.

curl POST /indexName-2017.06.10/_flush

Close : Close indexes persist in elasticsearch server but not available for searching. For making Index open we can use below command in curl .

curl POST /indexName-2017.06.10/_close

Delete : Delete operation on index will delete completely from server.

curl POST /indexName-2017.06.10/_delete

Now our code is ready to take care of indexes based on configured time and test . we test it after running above code.

Below steps are for making your index manager code auto runnable in Linux environment.

Create Auto Runnable Jar

Export ElasticsearchAutoIndexManager project as auto runnable jar by selecting as Launch class ElascticsearchIndexManagerClient. To learn about Auto runnable jar creation steps following link How to make and auto run /executable jar with dependencies?

Create Script File to Execute  Autorun Jar

Create script file as below with name as IndexManger.sh and save it.

#!/bin/bash
~/JAVA/jdk1.8.0_66/bin/java  -jar /opt/app/facingissuesonit/automate/IndexManagerClient.jar

Create Cron Tab configuration for schedule and receive email alert

Linux provide cron tab for executing schedule job/scripts. by using cron tab will execute  runnable jar by using above script file

  • Use command crontab -e to make and edit existing entries in cron tab.
  • Make below cron entry in this editor  for executing IndexManager.sh script on every night 1AM.
  • If you want to get execution alert to you and your team  with console logs also add your email id as below.
  • Save cron tab as ESC then (:wq)

Below are some more example for cron tab expression.

0 * * * *           : Run Every hour of day
* * * * *           : Every minute of day
30 4 * * *         : Run on 4:30 AM everyday
5 10,22 * * *   : Run twice on 10:05 and 22:05
5 0 * * *          : Run after Midnight

Read More

To read more on Elasticsearch REST , sample clients, configurations with example follow link Elasticsearch REST Tutorial and Elasticsearch Issues.

Hope this blog was helpful for you.

Leave you feedback to enhance more on this topic so that make it more helpful for others.