Using OSGi console to debug things

This OSGi console was very helpful to me in finding out problems relevant to OSGi bundles. You can have lot of information about this in the Internet. In here what I am going to write is about how I used it in the debugging.

First start the server with following option

-DosgiConsole. This will start the server with OSGi console. This is very helpful in situations like, checking whether all bundles that you put in to droppings have activated. When the server starts. Put ss in console. This will give a list of all bundles.

Use it with 'like' modifier to get the relevant bundles only. Ex ss like student

Now you can see state of all bundles. With the installation of a bundle in the OSGi runtime this bundle is persisted in a local bundle cache. The OSGi runtime then tries to resolve all dependencies of the bundle.If all required dependencies are resolved the bundle is in the status RESOLVED otherwise it is in the status INSTALLED.

If several bundles exist which would satisfy the dependency, then the bundle with the highest version is used. If the versions are the same, then the bundle with the lowest ID will be used. If the bundle is started, its status is STARTING. Afterwards it gets the ACTIVE status. - (http://www.vogella.com/articles/OSGi/article.html#osgiarch_bundles good reference)

You have to have all your bundles acticvated. If not check what is wrong with them using 'bundle <bundle number>' command. If there is no error, try starting it manually using 'start <bundle number>'

Publishing to BAM

I created this just to publish data I collect to BAM, so this do not adhere to good programming practices.

package org.wso2.carbon.usage.agent.util;

import java.net.MalformedURLException;
import java.net.SocketException;
import java.net.UnknownHostException;

import javax.security.sasl.AuthenticationException;

import org.wso2.carbon.eventbridge.agent.thrift.Agent;
import org.wso2.carbon.eventbridge.agent.thrift.DataPublisher;
import org.wso2.carbon.eventbridge.agent.thrift.conf.AgentConfiguration;
import org.wso2.carbon.eventbridge.agent.thrift.exception.AgentException;
import org.wso2.carbon.eventbridge.commons.Event;
import org.wso2.carbon.eventbridge.commons.exception.DifferentStreamDefinitionAlreadyDefinedException;
import org.wso2.carbon.eventbridge.commons.exception.MalformedStreamDefinitionException;
import org.wso2.carbon.eventbridge.commons.exception.NoStreamDefinitionExistException;
import org.wso2.carbon.eventbridge.commons.exception.StreamDefinitionException;
import org.wso2.carbon.eventbridge.commons.exception.TransportException;

public class PublishUtil2 {
    public static final String STREAM_NAME1 = "org.wso2.db6.kpiii";
    public static final String VERSION1 = "1.0.6";
    private static String streamId1;
    private static DataPublisher dataPublisher = null;
    
    

    public static void publish(long exceededBytes, long databasesize, String tenentdomain) throws AgentException, MalformedStreamDefinitionException,
    StreamDefinitionException, DifferentStreamDefinitionAlreadyDefinedException, MalformedURLException,
    AuthenticationException, NoStreamDefinitionExistException,
    org.wso2.carbon.eventbridge.commons.exception.AuthenticationException, TransportException, SocketException, UnknownHostException{
        
     System.out.println("Starting BAM KPI Agent");
        AgentConfiguration agentConfiguration = new AgentConfiguration();
        String currentDir = System.getProperty("user.dir");
        System.setProperty("javax.net.ssl.trustStore", currentDir + "/repository/resources/security/client-truststore.jks");
        System.setProperty("javax.net.ssl.trustStorePassword", "wso2carbon");
        Agent agent = new Agent(agentConfiguration);
        
     dataPublisher = null;
        try {
        dataPublisher = new DataPublisher("tcp://10.100.3.80:7613", "admin", "admin", agent);
        } catch (Throwable e){
         e.printStackTrace();
        }

        streamId1 = null;


        try {
            streamId1 = dataPublisher.findEventStream(STREAM_NAME1, VERSION1);
            System.out.println("Stream already defined");

        } catch (NoStreamDefinitionExistException e) {
            streamId1 = dataPublisher.defineEventStream("{" +
                    "  'name':'" + STREAM_NAME1 + "'," +
                    "  'version':'" + VERSION1 + "'," +
                    "  'nickName': 'DSSUsage'," +
                    "  'description': 'Exceeded DB Use'," +
                    "  'metaData':[" +
                    "          {'name':'clientType','type':'STRING'}" +
                    "  ]," +
                    "  'payloadData':[" +
                    "          {'name':'exceededBytes','type':'LONG'}," +
                    "          {'name':'databasesize','type':'LONG'}," +
                    "          {'name':'tenentdomain','type':'STRING'}" +
                    "  ]" +
                    "}");
        }
        //Publish event for a valid stream
        if (!streamId1.isEmpty()) {
            System.out.println("Stream ID: " + streamId1);
            publishEvents(tenentdomain, exceededBytes, databasesize);

//            for (int i = 0; i < 1; i++) {
//                publishEvents("malinga");
//                System.out.println("Events published : " + (i + 1) * 2);
//            }
//            try {
//                Thread.sleep(3000);
//            } catch (InterruptedException e) {
//            }

            dataPublisher.stop();
        }
    }
    
    public static void publishEvents(String name, long exceededBytes, long databasesize) throws AgentException {
     System.out.println(name);
     publishEvents(dataPublisher, streamId1, name, exceededBytes, databasesize);

    }

    
    public static void publishEvents(DataPublisher dataPublisher, String streamId, String name, long exceededBytes, long databasesize) throws AgentException {
        Event eventOne = new Event(streamId, System.currentTimeMillis(), new Object[]{"external"}, null,
                new Object[]{exceededBytes, databasesize, 3600.0, name});
        dataPublisher.publish(eventOne);

    }

}

Problems I had to face.

I tried to change the streamId1, but it was not possible. It gave a error in BAM side. Then I got to know that schema is saved under the STREAM_NAME1, and if I want to change it, I have to do that with a change in STREAM_NAME1 too.

There was no way to check the the published data as it would show some rubbish, in the data viewer in BAM. I got a nice client for one of my mentors that can get the data from Cassandra cluster. This is written by 'Shariq Muhammed', he is a software engineer at WSO2. This is also written just to read the data and he haven't thought much about adhering to good programming practices. You can have it from below link

https://svn.wso2.org/repos/wso2/people/shariq/BAMStatClient/

First I couldn't sent long, It took me a while to figure that out. It was because the number I send was taken as a int. I had to add the 'L' to end of it to get it working.

OSGi Services

Full system(carbon core + products) is built as a combination of OSGi bundles. Some of theses bundles expose services at OSGi level. I have to use them to get the tenant usage plan.

There is a service in manager which gives the usage plan of the tenant. To use it first I have to catch them, unluckily those manager component that is needed for the service it not there in the DSS, so we have to add them to DSS temporary as a feature. core and mgt are placed in DSS as a feature. First we thought is putting those to into droppings. But it didn't work as mgt package tried to start before core which is not possible as it requires core. Because of that we have to make it as a feature.

After putting them we have catch that service. Those are normally catches at serviceComponents(one class with 'activate' method). Below is how I did it. There are some methods that is used to bind and unbind the service objects we have to have them where we catch the service

I added the osgi comments as below  

/** 
 * @scr.component name="org.wso2.carbon.rssmanager" immediate="true" 
 * @scr.reference name="default.tenant.billing.service" 
 *                interface="org.wso2.carbon.stratos.common.TenantBillingService" 
 *                cardinality="1..1" policy="dynamic" 
 *                bind="setTenantBillingService" 
 *                unbind="unsetTenantBillingService" 
 * @scr.reference name="user.realmservice.default" 
 *                  interface="org.wso2.carbon.user.core.service.RealmService" 
 *                  cardinality="1..1" policy="dynamic" 
 *                  bind="setRealmService" 
 *                  unbind="unsetRealmService" 
 */ 

then I added following methods

/** 
     * osgi bind method for RealmService 
     * @param tenantBillingService 
     */ 
    protected void setRealmService(RealmService realmService) { 
        this.realmService = realmService; 
    } 

    /** 
     * osgi unbind method for RealmService 
     * @param tenantBillingService 
     */ 
    protected void unsetRealmService(RealmService realmService) { 
        this.realmService =null; 
    } 
   
    /** 
     * returns a RealmService objects that is used to get the tenant list. 
     * @param tenantBillingService 
     */ 
    private RealmService getRealmService() { 
        return realmService; 
    }

After catching it I can use it as below,

if(realmService!=null){ 
        TenantManager tenantManager = getRealmService().getTenantManager(); 
        Tenant[] tenants; 
        try { 
            tenants = (Tenant[]) tenantManager.getAllTenants(); 
            for (int i = 0; i < tenants.length; i++) { 
               // check for each tenant 
            } 
        } catch (UserStoreException e1) { 
            // TODO Auto-generated catch block 
            e1.printStackTrace(); 
        } 
        }else{ 
            System.out.println("realmService NULL"); 
        }

Problems came across Service was not available in DSS First I tried to catch this in my own java class (not in RSSManagerServiceComponent) which was OK but was not the standard. Standard is to catch it in RSSManagerServiceComponent. Anyway it should work, on matter how we do it. In my case it didn't work for tenant.billing.service . Problem was same service (tenant.billing.service not realmservice) was caught in RSSManagerServiceComponent before coming in to my class. This is where I figured out the importance of that standard. Had to include lot of other jar files in droppings.

Remote debugging with eclipce, for WSO2 Data service Server (will work for any server).

As I am working with RSS manager , it gave me some errors. Only way I knew to debug this is using print statements. But with sysouts I couldn't figure out the error. I had a small doubt whether one variable is not initialized. So need a way to check this, I found this remote debugging option which felt like home. It was like debugging a small app on the IDE.

This is how you can do it.

Go to debug configuration (just right click in the project Debug As>Debug configuration)

There go to 'Remote Java Application'

fill it like below

Run the server with given port as debug port

./wso2server.sh --debug 8000

put some breakpoints and you are good to go...

1st Verification – Verification of “index length + data length = actual disk space used”

I think I found where DBs are stored. As I am working in XAMPP there were stored in /opt/lampp/var/mysql there where folders for each database. When tried to open them it gives me a error saying “you don't have permission” Next thing I tried was trying to open it in the terminal using sudo. Unlucky me it gave me a error saying “sudo: cd: command not found”. Have to find a way to open such folders. I'll write a post on this, If I find a way (Or any one who know a way can help me by commenting below). Till then I used “sudo nautilus path/”

There were 3 files for each table in each db. A .frm file, .myd file, .myi file. All three are contributing to the space used

• FRM files contain table structure information.
  

• MYD contains the actual table data.
  
• MYI files contain the indexes.
  

Using “sudo ls -hs path” list the files inside the folder given by the path with sizes.



-s, --summarize (display only a total for each argument)
-h, --human-readable ( print sizes in human readable format (e.g., 1K 234M 2G))



so It showed that figures given in the information schema is similar to the results given by the above command. But size of the FRM file was not take in to account by me. But looks like all the .frm files are having the same length (12K) so I can sum them up If I know the number of tables. However I have to check why others ignore this file when they are calculating.

Some databases only have .frm file

Going through the database folders, I saw that some folders only have a .frm file in it. Searching for that I found out that there is two major engines within MySQL: MyISAM and InnoDB. If the table belongs to InnoDB, it only includes .frm file. Data of that kind of DBs are stored in a single or multiple (you can configure that) .idb files.

Prototype version 1 has to be verified.

This prototype is tested on my own machine with several MySQL users created by me. So this is not guaranteed to work in the real scenario. This should work with the actual billing architecture, to be any use for my work. This is the second and biggest verification that I have to do.

Before that there is a simple and basic verification that I have to consider. In prototype code I calculate a table size as follows (here information from information_schema.TABLES is used)

Table Size = Data Length + Index Length

Both Data Length and Index Length can be found in information_schema.TABLES. I have to verify that figure given by the above calculation is the correct table size.

Prototype version 1

As I was asked to build a proof of concept(POC), as I mentioned in before posts, I started working on it. By now I have a working prototype that logs the DB size information. Here I will create two tables one contain user details like user name, user DB size limit and binary information column 'exceeded' that will indicate whether subject user has exceeded his limits (this column is not used feather in my currant implementation, but I thought it would be helpful to have such a column in future).

Second table is a logging table, this logs the disk space used by each user, time to time. This again has 3 columns. A time stamp, user name and exceeded number of bytes (used number of bytes-user limit in bytes) are the information that is included in that table.

Below is the simple code that do that job

include code here.

import java.sql.*;

public class sizeCheck {

 public static void main(String args[]) {
  Connection con = null;
  Statement stat, stat2;
  boolean debug = true;
  try {
   Class.forName("com.mysql.jdbc.Driver").newInstance();
   con = DriverManager.getConnection("jdbc:mysql://localhost/", "root", "root");

   if (debug && !con.isClosed())
    System.out.println("Successfully connected to " + "MySQL server using TCP/IP...");

   stat = con.createStatement();
   stat2 = con.createStatement();

   // query to select all of the data from a table
   String selectQuery = "SELECT user,sizelimit FROM `Quota`.`Quota`";
   // get the results
   ResultSet results = stat.executeQuery(selectQuery);

   // output the results
   while (results.next()) {
    String getDBsizes =
                        "SELECT SUM(DATA_LENGTH) as sumData, SUM(INDEX_LENGTH) as sumIndex FROM `information_schema`.`TABLES` WHERE TABLE_SCHEMA LIKE '" +
                                results.getString("user") + "%'";
    ResultSet sizeResult = stat2.executeQuery(getDBsizes);
    sizeResult.next();
    if (debug) {
     System.out.println(sizeResult.getInt("sumData"));
     System.out.println(sizeResult.getInt("sumIndex"));
     System.out.println(sizeResult.getInt("sumData") + sizeResult.getInt("sumIndex"));
    }
    if (sizeResult.getInt("sumData") + sizeResult.getInt("sumIndex") > results.getInt("sizelimit")) {
     int exccededGB =
                      sizeResult.getInt("sumData") + sizeResult.getInt("sumIndex") -
                              results.getInt("sizelimit");
     String setExceeded =
                          "UPDATE `Quota`.`Quota` SET `exceeded` = '1' WHERE `Quota`.`user` = '" +
                                  results.getString("user") + "';";
     stat2.executeUpdate(setExceeded);
     String logExceeded =
                          "INSERT INTO `Quota`.`exceed` (`datetime` ,`user` ,`exceedbytes` ,`other`) VALUES (CURRENT_TIMESTAMP , '" +
                                  results.getString("user") +
                                  "', '" +
                                  exccededGB +
                                  "', '');";
     stat2.execute(logExceeded);
     // String revoke =
     // "REVOKE INSERT, UPDATE, CREATE, ALTER ON `"+
     // results.getString("user") +"\\_%` . * FROM '"+
     // results.getString("user") +"'@'localhost';";
     // if(debug)
     // System.out.println(revoke);
     // stat2.execute(revoke);
    } else {

     String setExceeded =
                          "UPDATE `Quota`.`Quota` SET `exceeded` = '0' WHERE `Quota`.`user` = '" +
                                  results.getString("user") + "';";
     stat2.executeUpdate(setExceeded);
     // String grant = "GRANT ALL PRIVILEGES ON `"+
     // results.getString("user") +"\\_%` . * TO '"+
     // results.getString("user") +"'@'localhost';";
     // if(debug)
     // System.out.println(grant);
     // stat2.execute(grant);
    }
   }

  } catch (Exception e) {
   System.err.println("Exception: " + e.getMessage());
  } finally {
   try {
    if (con != null)
     con.close();
   } catch (SQLException e) {
   }
  }

 }

}

you can download it form below

https://docs.google.com/open?id=0B4VQdLMBav1WNVFDaWxLRHNjWkU