Felix Framework Service Registry performance improvements

Implemented through the use of Java 5 concurrency APIs.
Unit tests included for the code that I changed.


git-svn-id: https://svn.apache.org/repos/asf/felix/trunk@1679327 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/framework/pom.xml b/framework/pom.xml
index 3d8e17f..515a589 100644
--- a/framework/pom.xml
+++ b/framework/pom.xml
@@ -144,5 +144,11 @@
         <version>4.2</version>
         <scope>test</scope>
     </dependency>
+    <dependency>
+        <groupId>org.mockito</groupId>
+        <artifactId>mockito-all</artifactId>
+        <version>1.10.19</version>
+        <scope>test</scope>
+    </dependency>
   </dependencies>
 </project>
diff --git a/framework/src/main/java/org/apache/felix/framework/ServiceRegistrationImpl.java b/framework/src/main/java/org/apache/felix/framework/ServiceRegistrationImpl.java
index ed22354..1d567a1 100644
--- a/framework/src/main/java/org/apache/felix/framework/ServiceRegistrationImpl.java
+++ b/framework/src/main/java/org/apache/felix/framework/ServiceRegistrationImpl.java
@@ -65,8 +65,8 @@
     private final ServiceReferenceImpl m_ref;
     // Flag indicating that we are unregistering.
     private volatile boolean m_isUnregistering = false;
-
-    private volatile Thread lock;
+    // This threadlocal is used to detect cycles.
+    private final ThreadLocal<Boolean> m_threadLoopDetection = new ThreadLocal<Boolean>();
 
     private final Object syncObject = new Object();
 
@@ -90,7 +90,7 @@
         m_ref = new ServiceReferenceImpl();
     }
 
-    protected synchronized boolean isValid()
+    protected boolean isValid()
     {
         return (m_svcObj != null);
     }
@@ -758,35 +758,18 @@
         }
     }
 
-    public boolean isLocked()
+    boolean currentThreadMarked()
     {
-        return this.lock == Thread.currentThread();
+        return m_threadLoopDetection.get() != null;
     }
 
-    public void lock()
+    void markCurrentThread()
     {
-        synchronized ( this.syncObject )
-        {
-            while ( this.lock != null )
-            {
-                try
-                {
-                    this.syncObject.wait();
-                }
-                catch ( final InterruptedException re) {
-                    // nothing to do
-                }
-            }
-            this.lock = Thread.currentThread();
-        }
+        m_threadLoopDetection.set(Boolean.TRUE);
     }
 
-    public void unlock()
+    void unmarkCurrentThread()
     {
-        synchronized ( this.syncObject )
-        {
-            this.lock = null;
-            this.syncObject.notifyAll();
-        }
+        m_threadLoopDetection.set(null);
     }
 }
\ No newline at end of file
diff --git a/framework/src/main/java/org/apache/felix/framework/ServiceRegistry.java b/framework/src/main/java/org/apache/felix/framework/ServiceRegistry.java
index 9fdad0d..96fe8ba 100644
--- a/framework/src/main/java/org/apache/felix/framework/ServiceRegistry.java
+++ b/framework/src/main/java/org/apache/felix/framework/ServiceRegistry.java
@@ -27,7 +27,10 @@
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.felix.framework.capabilityset.CapabilitySet;
 import org.apache.felix.framework.capabilityset.SimpleFilter;
@@ -112,7 +115,7 @@
         final Bundle bundle,
         final String[] classNames,
         final Object svcObj,
-        final Dictionary dict)
+        final Dictionary<?,?> dict)
     {
         // Create the service registration.
         final ServiceRegistrationImpl reg = new ServiceRegistrationImpl(
@@ -150,12 +153,6 @@
         // If this is a hook, it should be removed.
         this.hookRegistry.removeHooks(reg.getReference());
 
-        // Note that we don't lock the service registration here using
-        // the m_lockedRegsMap because we want to allow bundles to get
-        // the service during the unregistration process. However, since
-        // we do remove the registration from the service registry, no
-        // new bundles will be able to look up the service.
-
         // Now remove the registered service.
         final List<ServiceRegistration<?>> regs = m_regsMap.get(bundle);
         if (regs != null)
@@ -197,7 +194,7 @@
             {
                 if (usages[x].m_ref.equals(ref))
                 {
-                    ungetService(clients[i], ref, (usages[x].m_prototype ? usages[x].m_svcObj : null));
+                    ungetService(clients[i], ref, (usages[x].m_prototype ? usages[x].getService() : null));
                 }
             }
         }
@@ -300,7 +297,7 @@
             ((ServiceRegistrationImpl.ServiceReferenceImpl) ref).getRegistration();
 
         // We don't allow cycles when we call out to the service factory.
-        if ( reg.isLocked() )
+        if ( reg.currentThreadMarked() )
         {
             throw new ServiceException(
                     "ServiceFactory.getService() resulted in a cycle.",
@@ -308,63 +305,75 @@
                     null);
         }
 
-        // no concurrent operations on the same service registration
-        reg.lock();
-        // Make sure the service registration is still valid.
-        if (reg.isValid())
-        {
-            // Get the usage count, if any.
-            // if prototype, we always create a new usage
-            usage = isPrototype ? null : getUsageCount(bundle, ref, null);
-
-            // If we don't have a usage count, then create one and
-            // since the spec says we increment usage count before
-            // actually getting the service object.
-            if (usage == null)
-            {
-                usage = addUsageCount(bundle, ref, isPrototype);
-            }
-
-            // Increment the usage count and grab the already retrieved
-            // service object, if one exists.
-            usage.m_count++;
-            svcObj = usage.m_svcObj;
-            if ( isServiceObjects )
-            {
-                usage.m_serviceObjectsCount++;
-            }
-        }
-
-        // If we have a usage count, but no service object, then we haven't
-        // cached the service object yet, so we need to create one now without
-        // holding the lock, since we will potentially call out to a service
-        // factory.
         try
         {
-            if ((usage != null) && (svcObj == null))
+            reg.markCurrentThread();
+
+            // Make sure the service registration is still valid.
+            if (reg.isValid())
             {
-                svcObj = reg.getService(bundle);
+                // Get the usage count, or create a new one. If this is a
+                // prototype, the we'll alway create a new one.
+                usage = obtainUsageCount(bundle, ref, null, isPrototype);
+
+                // Increment the usage count and grab the already retrieved
+                // service object, if one exists.
+                usage.m_count.incrementAndGet();
+                svcObj = usage.getService();
+                if ( isServiceObjects )
+                {
+                    usage.m_serviceObjectsCount.incrementAndGet();
+                }
+
+                // If we have a usage count, but no service object, then we haven't
+                // cached the service object yet, so we need to create one.
+                if ((usage != null) && (svcObj == null))
+                {
+                    ServiceHolder holder = null;
+
+                    // There is a possibility that the holder is unset between the compareAndSet() and the get()
+                    // below. If that happens get() returns null and we may have to set a new holder. This is
+                    // why the below section is in a loop.
+                    while (holder == null)
+                    {
+                        ServiceHolder h = new ServiceHolder();
+                        if (usage.m_svcHolderRef.compareAndSet(null, h))
+                        {
+                            holder = h;
+                            svcObj = reg.getService(bundle);
+                            holder.m_service = svcObj;
+                            holder.m_latch.countDown();
+                        }
+                        else
+                        {
+                            holder = usage.m_svcHolderRef.get();
+                            if (holder != null)
+                            {
+                                try
+                                {
+                                    // Need to ensure that the other thread has obtained
+                                    // the service.
+                                    holder.m_latch.await();
+                                }
+                                catch (InterruptedException e)
+                                {
+                                    throw new RuntimeException(e);
+                                }
+                                svcObj = holder.m_service;
+                            }
+                        }
+                    }
+                }
             }
         }
         finally
         {
-            // If we successfully retrieved a service object, then we should
-            // cache it in the usage count. If not, we should flush the usage
-            // count. Either way, we need to unlock the service registration
-            // so that any threads waiting for it can continue.
+            reg.unmarkCurrentThread();
 
-            // Before caching the service object, double check to see if
-            // the registration is still valid, since it may have been
-            // unregistered while we didn't hold the lock.
             if (!reg.isValid() || (svcObj == null))
             {
                 flushUsageCount(bundle, ref, usage);
             }
-            else
-            {
-                usage.m_svcObj = svcObj;
-            }
-            reg.unlock();
         }
 
         return (S) svcObj;
@@ -375,73 +384,66 @@
         final ServiceRegistrationImpl reg =
             ((ServiceRegistrationImpl.ServiceReferenceImpl) ref).getRegistration();
 
-        if ( reg.isLocked() )
+        if ( reg.currentThreadMarked() )
         {
             throw new IllegalStateException(
                     "ServiceFactory.ungetService() resulted in a cycle.");
         }
 
-        UsageCount usage = null;
-
-        // First make sure that no existing operation is currently
-        // being performed by another thread on the service registration.
-        reg.lock();
-
-        // Get the usage count.
-        usage = getUsageCount(bundle, ref, svcObj);
-        // If there is no cached services, then just return immediately.
-        if (usage == null)
-        {
-            reg.unlock();
-            return false;
-        }
-        // if this is a call from service objects and the service was not fetched from
-        // there, return false
-        if ( svcObj != null )
-        {
-            // TODO have a proper conditional decrement and get, how???
-            usage.m_serviceObjectsCount--;
-            if (usage.m_serviceObjectsCount < 0)
-            {
-                reg.unlock();
-                return false;
-            }
-        }
-
-        // If usage count will go to zero, then unget the service
-        // from the registration; we do this outside the lock
-        // since this might call out to the service factory.
         try
         {
-            if (usage.m_count == 1)
+            // Mark the current thread to avoid cycles
+            reg.markCurrentThread();
+
+            // Get the usage count.
+            UsageCount usage = obtainUsageCount(bundle, ref, svcObj, null);
+            // If there are no cached services, then just return immediately.
+            if (usage == null)
             {
-                // Remove reference from usages array.
-                ((ServiceRegistrationImpl.ServiceReferenceImpl) ref)
-                    .getRegistration().ungetService(bundle, usage.m_svcObj);
+                return false;
+            }
+            // if this is a call from service objects and the service was not fetched from
+            // there, return false
+            if ( svcObj != null )
+            {
+                if (usage.m_serviceObjectsCount.decrementAndGet() < 0)
+                {
+                    return false;
+                }
+            }
+
+            // If usage count will go to zero, then unget the service
+            // from the registration.
+            try
+            {
+                if (usage.m_count.get() == 1)
+                {
+                    // Remove reference from usages array.
+                    ((ServiceRegistrationImpl.ServiceReferenceImpl) ref)
+                        .getRegistration().ungetService(bundle, usage.getService());
+                }
+            }
+            finally
+            {
+                // Finally, decrement usage count and flush if it goes to zero or
+                // the registration became invalid.
+
+                // Decrement usage count, which spec says should happen after
+                // ungetting the service object.
+                int c = usage.m_count.decrementAndGet();
+
+                // If the registration is invalid or the usage count has reached
+                // zero, then flush it.
+                if ((c <= 0) || !reg.isValid())
+                {
+                    usage.m_svcHolderRef.set(null);
+                    flushUsageCount(bundle, ref, usage);
+                }
             }
         }
         finally
         {
-            // Finally, decrement usage count and flush if it goes to zero or
-            // the registration became invalid while we were not holding the
-            // lock. Either way, unlock the service registration so that any
-            // threads waiting for it can continue.
-
-            // Decrement usage count, which spec says should happen after
-            // ungetting the service object.
-            usage.m_count--;
-
-            // If the registration is invalid or the usage count has reached
-            // zero, then flush it.
-            if (!reg.isValid() || (usage.m_count <= 0))
-            {
-                usage.m_svcObj = null;
-                flushUsageCount(bundle, ref, usage);
-            }
-
-            // Release the registration lock so any waiting threads can
-            // continue.
-            reg.unlock();
+            reg.unmarkCurrentThread();
         }
 
         return true;
@@ -471,7 +473,7 @@
         for (int i = 0; i < usages.length; i++)
         {
             // Keep ungetting until all usage count is zero.
-            while (ungetService(bundle, usages[i].m_ref, usages[i].m_prototype ? usages[i].m_svcObj : null))
+            while (ungetService(bundle, usages[i].m_ref, usages[i].m_prototype ? usages[i].getService() : null))
             {
                 // Empty loop body.
             }
@@ -508,7 +510,7 @@
         return bundles;
     }
 
-    void servicePropertiesModified(ServiceRegistration<?> reg, Dictionary oldProps)
+    void servicePropertiesModified(ServiceRegistration<?> reg, Dictionary<?,?> oldProps)
     {
         this.hookRegistry.updateHooks(reg.getReference());
         if (m_callbacks != null)
@@ -524,56 +526,66 @@
     }
 
     /**
-     * Utility method to retrieve the specified bundle's usage count for the
-     * specified service reference.
-     * @param bundle The bundle whose usage counts are being searched.
-     * @param ref The service reference to find in the bundle's usage counts.
-     * @return The associated usage count or null if not found.
-    **/
-    private UsageCount getUsageCount(Bundle bundle, ServiceReference<?> ref, final Object svcObj)
+     * Obtain a UsageCount object, by looking for an existing one or creating a new one (if possible).
+     * This method tries to find a UsageCount object in the {@code m_inUseMap}. If one is found then
+     * this is returned, otherwise a UsageCount object will be created, but this can only be done if
+     * the {@code isPrototype} parameter is not {@code null}. If {@code isPrototype} is {@code TRUE}
+     * then a new UsageCount object will always be created.
+     * @param bundle The bundle using the service.
+     * @param ref The Service Reference.
+     * @param svcObj A Service Object, if applicable.
+     * @param isPrototype {@code TRUE} if we know that this is a prototype, {@ FALSE} if we know that
+     * it isn't. There are cases where we don't know (the pure lookup case), in that case use {@code null}.
+     * @return The UsageCount object if it could be obtained, or {@code null} otherwise.
+     */
+    UsageCount obtainUsageCount(Bundle bundle, ServiceReference<?> ref, Object svcObj, Boolean isPrototype)
     {
-        UsageCount[] usages = m_inUseMap.get(bundle);
-        for (int i = 0; (usages != null) && (i < usages.length); i++)
+        UsageCount usage = null;
+
+        // This method uses an optimistic concurrency mechanism with a conditional put/replace
+        // on the m_inUseMap. If this fails (because another thread made changes) this thread
+        // retries the operation. This is the purpose of the while loop.
+        boolean success = false;
+        while (!success)
         {
-            if (usages[i].m_ref.equals(ref)
-               && ((svcObj == null && !usages[i].m_prototype) || usages[i].m_svcObj == svcObj))
+            UsageCount[] usages = m_inUseMap.get(bundle);
+
+            // If we know it's a prototype, then we always need to create a new usage count
+            if (!Boolean.TRUE.equals(isPrototype))
             {
-                return usages[i];
+                for (int i = 0; (usages != null) && (i < usages.length); i++)
+                {
+                    if (usages[i].m_ref.equals(ref)
+                       && ((svcObj == null && !usages[i].m_prototype) || usages[i].getService() == svcObj))
+                    {
+                        return usages[i];
+                    }
+                }
+            }
+
+            // We haven't found an existing usage count object so we need to create on. For this we need to
+            // know whether this is a prototype or not.
+            if (isPrototype == null)
+            {
+                // If this parameter isn't passed in we can't create a usage count.
+                return null;
+            }
+
+            // Add a new Usage Count.
+            usage = new UsageCount(ref, isPrototype);
+            if (usages == null)
+            {
+                UsageCount[] newUsages = new UsageCount[] { usage };
+                success = m_inUseMap.putIfAbsent(bundle, newUsages) == null;
+            }
+            else
+            {
+                UsageCount[] newUsages = new UsageCount[usages.length + 1];
+                System.arraycopy(usages, 0, newUsages, 0, usages.length);
+                newUsages[usages.length] = usage;
+                success = m_inUseMap.replace(bundle, usages, newUsages);
             }
         }
-        return null;
-    }
-
-    /**
-     * Utility method to update the specified bundle's usage count array to
-     * include the specified service. This method should only be called
-     * to add a usage count for a previously unreferenced service. If the
-     * service already has a usage count, then the existing usage count
-     * counter simply needs to be incremented.
-     * @param bundle The bundle acquiring the service.
-     * @param ref The service reference of the acquired service.
-     * @param svcObj The service object of the acquired service.
-    **/
-    private UsageCount addUsageCount(Bundle bundle, ServiceReference<?> ref, boolean isPrototype)
-    {
-        UsageCount[] usages = m_inUseMap.get(bundle);
-
-        UsageCount usage = new UsageCount(ref, isPrototype);
-
-        if (usages == null)
-        {
-            usages = new UsageCount[] { usage };
-        }
-        else
-        {
-            UsageCount[] newUsages = new UsageCount[usages.length + 1];
-            System.arraycopy(usages, 0, newUsages, 0, usages.length);
-            newUsages[usages.length] = usage;
-            usages = newUsages;
-        }
-
-        m_inUseMap.put(bundle, usages);
-
         return usage;
     }
 
@@ -589,41 +601,51 @@
      * @param bundle The bundle whose usage count should be removed.
      * @param ref The service reference whose usage count should be removed.
     **/
-    private void flushUsageCount(Bundle bundle, ServiceReference<?> ref, UsageCount uc)
+    void flushUsageCount(Bundle bundle, ServiceReference<?> ref, UsageCount uc)
     {
-        UsageCount[] usages = m_inUseMap.get(bundle);
-        for (int i = 0; (usages != null) && (i < usages.length); i++)
+        // This method uses an optimistic concurrency mechanism with conditional modifications
+        // on the m_inUseMap. If this fails (because another thread made changes) this thread
+        // retries the operation. This is the purpose of the while loop.
+        boolean success = false;
+        while (!success)
         {
-            if ((uc == null && usages[i].m_ref.equals(ref)) || (uc == usages[i]))
+            UsageCount[] usages = m_inUseMap.get(bundle);
+            final UsageCount[] orgUsages = usages;
+            for (int i = 0; (usages != null) && (i < usages.length); i++)
             {
-                // If this is the only usage, then point to empty list.
-                if ((usages.length - 1) == 0)
+                if ((uc == null && usages[i].m_ref.equals(ref)) || (uc == usages[i]))
                 {
-                    usages = null;
-                }
-                // Otherwise, we need to do some array copying.
-                else
-                {
-                    UsageCount[] newUsages = new UsageCount[usages.length - 1];
-                    System.arraycopy(usages, 0, newUsages, 0, i);
-                    if (i < newUsages.length)
+                    // If this is the only usage, then point to empty list.
+                    if ((usages.length - 1) == 0)
                     {
-                        System.arraycopy(
-                            usages, i + 1, newUsages, i, newUsages.length - i);
+                        usages = null;
                     }
-                    usages = newUsages;
-                    i--;
+                    // Otherwise, we need to do some array copying.
+                    else
+                    {
+                        UsageCount[] newUsages = new UsageCount[usages.length - 1];
+                        System.arraycopy(usages, 0, newUsages, 0, i);
+                        if (i < newUsages.length)
+                        {
+                            System.arraycopy(
+                                usages, i + 1, newUsages, i, newUsages.length - i);
+                        }
+                        usages = newUsages;
+                        i--;
+                    }
                 }
             }
-        }
 
-        if (usages != null)
-        {
-            m_inUseMap.put(bundle, usages);
-        }
-        else
-        {
-            m_inUseMap.remove(bundle);
+            if (usages == orgUsages)
+                return; // no change in map
+
+            if (orgUsages != null)
+            {
+                if (usages != null)
+                    success = m_inUseMap.replace(bundle, orgUsages, usages);
+                else
+                    success = m_inUseMap.remove(bundle, orgUsages);
+            }
         }
     }
 
@@ -632,25 +654,36 @@
         return this.hookRegistry;
     }
 
-    private static class UsageCount
+    static class UsageCount
     {
-        public final ServiceReference<?> m_ref;
-        public final boolean m_prototype;
+        final ServiceReference<?> m_ref;
+        final boolean m_prototype;
 
-        public volatile int m_count;
-        public volatile int m_serviceObjectsCount;
-
-        public volatile Object m_svcObj;
+        final AtomicInteger m_count = new AtomicInteger();
+        final AtomicInteger m_serviceObjectsCount = new AtomicInteger();
+        final AtomicReference<ServiceHolder> m_svcHolderRef = new AtomicReference<ServiceHolder>();
 
         UsageCount(final ServiceReference<?> ref, final boolean isPrototype)
         {
             m_ref = ref;
             m_prototype = isPrototype;
         }
+
+        Object getService()
+        {
+            ServiceHolder sh = m_svcHolderRef.get();
+            return sh == null ? null : sh.m_service;
+        }
+    }
+
+    static class ServiceHolder
+    {
+        final CountDownLatch m_latch = new CountDownLatch(1);
+        volatile Object m_service;
     }
 
     public interface ServiceRegistryCallbacks
     {
-        void serviceChanged(ServiceEvent event, Dictionary oldProps);
+        void serviceChanged(ServiceEvent event, Dictionary<?,?> oldProps);
     }
 }
diff --git a/framework/src/test/java/org/apache/felix/framework/ServiceRegistryTest.java b/framework/src/test/java/org/apache/felix/framework/ServiceRegistryTest.java
index 3db085a..abab92c 100644
--- a/framework/src/test/java/org/apache/felix/framework/ServiceRegistryTest.java
+++ b/framework/src/test/java/org/apache/felix/framework/ServiceRegistryTest.java
@@ -18,15 +18,30 @@
  */
 package org.apache.felix.framework;
 
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Hashtable;
+import java.util.List;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import junit.framework.TestCase;
 
+import org.apache.felix.framework.ServiceRegistrationImpl.ServiceReferenceImpl;
+import org.apache.felix.framework.ServiceRegistry.ServiceHolder;
+import org.apache.felix.framework.ServiceRegistry.UsageCount;
 import org.easymock.MockControl;
+import org.mockito.AdditionalAnswers;
+import org.mockito.InOrder;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
 import org.osgi.framework.Bundle;
 import org.osgi.framework.BundleContext;
 import org.osgi.framework.ServiceEvent;
+import org.osgi.framework.ServiceException;
 import org.osgi.framework.ServiceFactory;
 import org.osgi.framework.ServiceReference;
 import org.osgi.framework.ServiceRegistration;
@@ -331,4 +346,703 @@
         assertEquals("Unregistration should have no effect", 0, sr.getHookRegistry().getHooks(FindHook.class).size());
         assertEquals("Unregistration should have no effect", 0, sr.getHookRegistry().getHooks(ListenerHook.class).size());
     }
+
+    @SuppressWarnings("unchecked")
+    public void testGetService()
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        String svc = "foo";
+
+        Bundle b = Mockito.mock(Bundle.class);
+        ServiceRegistrationImpl reg = Mockito.mock(ServiceRegistrationImpl.class);
+        Mockito.when(reg.isValid()).thenReturn(true);
+        Mockito.when(reg.getService(b)).thenReturn(svc);
+
+        ServiceReferenceImpl ref = Mockito.mock(ServiceReferenceImpl.class);
+        Mockito.when(ref.getRegistration()).thenReturn(reg);
+
+        assertSame(svc, sr.getService(b, ref, false));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testGetServiceHolderAwait() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        final String svc = "test";
+
+        Bundle b = Mockito.mock(Bundle.class);
+        ServiceRegistrationImpl reg = Mockito.mock(ServiceRegistrationImpl.class);
+        Mockito.when(reg.isValid()).thenReturn(true);
+
+        ServiceReferenceImpl ref = Mockito.mock(ServiceReferenceImpl.class);
+        Mockito.when(ref.getRegistration()).thenReturn(reg);
+
+        UsageCount uc = sr.obtainUsageCount(b, ref, null, false);
+
+        // Set an empty Service Holder so we can test that it waits.
+        final ServiceHolder sh = new ServiceHolder();
+        uc.m_svcHolderRef.set(sh);
+
+        final StringBuilder sb = new StringBuilder();
+        final AtomicBoolean threadException = new AtomicBoolean(false);
+        Thread t = new Thread() {
+            @Override
+            public void run()
+            {
+                try { Thread.sleep(250); } catch (InterruptedException e) {}
+                sh.m_service = svc;
+                if (sb.length() > 0)
+                {
+                    // Should not have put anything in SB until countDown() was called...
+                    threadException.set(true);
+                }
+                sh.m_latch.countDown();
+            }
+        };
+        assertFalse(t.isInterrupted());
+        t.start();
+
+        Object actualSvc = sr.getService(b, ref, false);
+        sb.append(actualSvc);
+
+        t.join();
+        assertFalse("This thread did not wait until the latch was count down",
+                threadException.get());
+
+        assertSame(svc, actualSvc);
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testGetServicePrototype() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        String svc = "xyz";
+
+        Bundle b = Mockito.mock(Bundle.class);
+        ServiceRegistrationImpl reg = Mockito.mock(ServiceRegistrationImpl.class);
+        Mockito.when(reg.isValid()).thenReturn(true);
+        Mockito.when(reg.getService(b)).thenReturn(svc);
+
+        ServiceReferenceImpl ref = Mockito.mock(ServiceReferenceImpl.class);
+        Mockito.when(ref.getRegistration()).thenReturn(reg);
+
+        assertSame(svc, sr.getService(b, ref, true));
+
+        final ConcurrentMap<Bundle, UsageCount[]> inUseMap =
+                (ConcurrentMap<Bundle, UsageCount[]>) getPrivateField(sr, "m_inUseMap");
+        UsageCount[] uca = inUseMap.get(b);
+        assertEquals(1, uca.length);
+        assertEquals(1, uca[0].m_serviceObjectsCount.get());
+
+        sr.getService(b, ref, true);
+        assertEquals(2, uca[0].m_serviceObjectsCount.get());
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testGetServiceThreadMarking() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        Bundle b = Mockito.mock(Bundle.class);
+        ServiceRegistrationImpl reg = Mockito.mock(ServiceRegistrationImpl.class);
+
+        ServiceReferenceImpl ref = Mockito.mock(ServiceReferenceImpl.class);
+        Mockito.when(ref.getRegistration()).thenReturn(reg);
+
+        sr.getService(b, ref, false);
+
+        InOrder inOrder = Mockito.inOrder(reg);
+        inOrder.verify(reg, Mockito.times(1)).currentThreadMarked();
+        inOrder.verify(reg, Mockito.times(1)).markCurrentThread();
+        inOrder.verify(reg, Mockito.times(1)).unmarkCurrentThread();
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testGetServiceThreadMarking2() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        String svc = "bar";
+
+        Bundle b = Mockito.mock(Bundle.class);
+
+        ServiceRegistrationImpl reg = (ServiceRegistrationImpl) sr.registerService(
+                b, new String [] {String.class.getName()}, svc, null);
+
+        ServiceReferenceImpl ref = Mockito.mock(ServiceReferenceImpl.class);
+        Mockito.when(ref.getRegistration()).thenReturn(reg);
+
+        reg.markCurrentThread();
+        try
+        {
+            sr.getService(b, ref, false);
+            fail("Should have thrown an exception to signal reentrant behaviour");
+        }
+        catch (ServiceException se)
+        {
+            assertEquals(ServiceException.FACTORY_ERROR, se.getType());
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testUngetService() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        Bundle b = Mockito.mock(Bundle.class);
+        ServiceRegistrationImpl reg = Mockito.mock(ServiceRegistrationImpl.class);
+
+        ServiceReferenceImpl ref = Mockito.mock(ServiceReferenceImpl.class);
+        Mockito.when(ref.getRegistration()).thenReturn(reg);
+
+        final ConcurrentMap<Bundle, UsageCount[]> inUseMap =
+                (ConcurrentMap<Bundle, UsageCount[]>) getPrivateField(sr, "m_inUseMap");
+
+        UsageCount uc = new UsageCount(ref, false);
+        uc.m_svcHolderRef.set(new ServiceHolder());
+
+        inUseMap.put(b, new UsageCount[] {uc});
+
+        assertTrue(sr.ungetService(b, ref, null));
+        assertNull(uc.m_svcHolderRef.get());
+        assertNull(inUseMap.get(b));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testUngetService2() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        Bundle b = Mockito.mock(Bundle.class);
+        ServiceRegistrationImpl reg = Mockito.mock(ServiceRegistrationImpl.class);
+
+        ServiceReferenceImpl ref = Mockito.mock(ServiceReferenceImpl.class);
+        Mockito.when(ref.getRegistration()).thenReturn(reg);
+
+        final ConcurrentMap<Bundle, UsageCount[]> inUseMap =
+                (ConcurrentMap<Bundle, UsageCount[]>) getPrivateField(sr, "m_inUseMap");
+
+        UsageCount uc = new UsageCount(ref, false);
+        uc.m_svcHolderRef.set(new ServiceHolder());
+        uc.m_count.incrementAndGet();
+
+        Mockito.verify(reg, Mockito.never()).
+            ungetService(Mockito.isA(Bundle.class), Mockito.any());
+        inUseMap.put(b, new UsageCount[] {uc});
+
+        assertTrue(sr.ungetService(b, ref, null));
+        assertNull(uc.m_svcHolderRef.get());
+        assertNull(inUseMap.get(b));
+
+        Mockito.verify(reg, Mockito.times(1)).
+        ungetService(Mockito.isA(Bundle.class), Mockito.any());
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testUngetService3() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        Bundle b = Mockito.mock(Bundle.class);
+        ServiceRegistrationImpl reg = Mockito.mock(ServiceRegistrationImpl.class);
+        Mockito.when(reg.isValid()).thenReturn(true);
+
+        ServiceReferenceImpl ref = Mockito.mock(ServiceReferenceImpl.class);
+        Mockito.when(ref.getRegistration()).thenReturn(reg);
+
+        final ConcurrentMap<Bundle, UsageCount[]> inUseMap =
+                (ConcurrentMap<Bundle, UsageCount[]>) getPrivateField(sr, "m_inUseMap");
+
+        UsageCount uc = new UsageCount(ref, false);
+        uc.m_svcHolderRef.set(new ServiceHolder());
+        uc.m_count.set(2);
+
+        inUseMap.put(b, new UsageCount[] {uc});
+
+        assertTrue(sr.ungetService(b, ref, null));
+        assertNotNull(uc.m_svcHolderRef.get());
+        assertNotNull(inUseMap.get(b));
+
+        Mockito.verify(reg, Mockito.never()).
+            ungetService(Mockito.isA(Bundle.class), Mockito.any());
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testUngetService4() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        Bundle b = Mockito.mock(Bundle.class);
+        ServiceRegistrationImpl reg = Mockito.mock(ServiceRegistrationImpl.class);
+        Mockito.when(reg.isValid()).thenReturn(false);
+
+        ServiceReferenceImpl ref = Mockito.mock(ServiceReferenceImpl.class);
+        Mockito.when(ref.getRegistration()).thenReturn(reg);
+
+        final ConcurrentMap<Bundle, UsageCount[]> inUseMap =
+                (ConcurrentMap<Bundle, UsageCount[]>) getPrivateField(sr, "m_inUseMap");
+
+        UsageCount uc = new UsageCount(ref, false);
+        uc.m_svcHolderRef.set(new ServiceHolder());
+        uc.m_count.set(2);
+
+        inUseMap.put(b, new UsageCount[] {uc});
+
+        assertTrue(sr.ungetService(b, ref, null));
+        assertNull(uc.m_svcHolderRef.get());
+        assertNull(inUseMap.get(b));
+
+        Mockito.verify(reg, Mockito.never()).
+            ungetService(Mockito.isA(Bundle.class), Mockito.any());
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testUngetService5() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        Bundle b = Mockito.mock(Bundle.class);
+        ServiceRegistrationImpl reg = Mockito.mock(ServiceRegistrationImpl.class);
+        Mockito.doThrow(new RuntimeException("Test!")).when(reg).
+            ungetService(Mockito.isA(Bundle.class), Mockito.any());
+
+        ServiceReferenceImpl ref = Mockito.mock(ServiceReferenceImpl.class);
+        Mockito.when(ref.getRegistration()).thenReturn(reg);
+
+        final ConcurrentMap<Bundle, UsageCount[]> inUseMap =
+                (ConcurrentMap<Bundle, UsageCount[]>) getPrivateField(sr, "m_inUseMap");
+
+        String svc = "myService";
+        UsageCount uc = new UsageCount(ref, false);
+        ServiceHolder sh = new ServiceHolder();
+        sh.m_service = svc;
+        sh.m_latch.countDown();
+        uc.m_svcHolderRef.set(sh);
+        uc.m_count.set(1);
+
+        inUseMap.put(b, new UsageCount[] {uc});
+
+        try
+        {
+            assertTrue(sr.ungetService(b, ref, null));
+            fail("Should have propagated the runtime exception");
+        }
+        catch (RuntimeException re)
+        {
+            assertEquals("Test!", re.getMessage());
+        }
+        assertNull(uc.m_svcHolderRef.get());
+        assertNull(inUseMap.get(b));
+
+        Mockito.verify(reg, Mockito.times(1)).ungetService(b, svc);
+    }
+
+    public void testUngetServiceThreadMarking()
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        Bundle b = Mockito.mock(Bundle.class);
+        ServiceRegistrationImpl reg = Mockito.mock(ServiceRegistrationImpl.class);
+
+        ServiceReferenceImpl ref = Mockito.mock(ServiceReferenceImpl.class);
+        Mockito.when(ref.getRegistration()).thenReturn(reg);
+
+        assertFalse("There is no usage count, so this method should return false",
+                sr.ungetService(b, ref, null));
+
+        InOrder inOrder = Mockito.inOrder(reg);
+        inOrder.verify(reg, Mockito.times(1)).currentThreadMarked();
+        inOrder.verify(reg, Mockito.times(1)).markCurrentThread();
+        inOrder.verify(reg, Mockito.times(1)).unmarkCurrentThread();
+    }
+
+    public void testUngetServiceThreadMarking2()
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        Bundle b = Mockito.mock(Bundle.class);
+        ServiceRegistrationImpl reg = Mockito.mock(ServiceRegistrationImpl.class);
+        Mockito.when(reg.currentThreadMarked()).thenReturn(true);
+
+        ServiceReferenceImpl ref = Mockito.mock(ServiceReferenceImpl.class);
+        Mockito.when(ref.getRegistration()).thenReturn(reg);
+
+        try
+        {
+            sr.ungetService(b, ref, null);
+            fail("The thread should be observed as marked and hence throw an exception");
+        }
+        catch (IllegalStateException ise)
+        {
+            // good
+        }
+    }
+
+    public void testObtainUsageCount() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        @SuppressWarnings("unchecked")
+        ConcurrentMap<Bundle, UsageCount[]> inUseMap = (ConcurrentMap<Bundle, UsageCount[]>) getPrivateField(sr, "m_inUseMap");
+
+        assertEquals("Precondition", 0, inUseMap.size());
+
+        Bundle b = Mockito.mock(Bundle.class);
+        ServiceReference<?> ref = Mockito.mock(ServiceReference.class);
+        UsageCount uc = sr.obtainUsageCount(b, ref, null, false);
+        assertEquals(1, inUseMap.size());
+        assertEquals(1, inUseMap.get(b).length);
+        assertSame(uc, inUseMap.get(b)[0]);
+        assertSame(ref, uc.m_ref);
+        assertFalse(uc.m_prototype);
+
+        UsageCount uc2 = sr.obtainUsageCount(b, ref, null, false);
+        assertSame(uc, uc2);
+
+        ServiceReference<?> ref2 = Mockito.mock(ServiceReference.class);
+        UsageCount uc3 = sr.obtainUsageCount(b, ref2, null, false);
+        assertNotSame(uc3, uc2);
+        assertSame(ref2, uc3.m_ref);
+    }
+
+    public void testObtainUsageCountPrototype() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        @SuppressWarnings("unchecked")
+        ConcurrentMap<Bundle, UsageCount[]> inUseMap = (ConcurrentMap<Bundle, UsageCount[]>) getPrivateField(sr, "m_inUseMap");
+
+        Bundle b = Mockito.mock(Bundle.class);
+        ServiceReference<?> ref = Mockito.mock(ServiceReference.class);
+        UsageCount uc = sr.obtainUsageCount(b, ref, null, true);
+        assertEquals(1, inUseMap.size());
+        assertEquals(1, inUseMap.values().iterator().next().length);
+
+        ServiceReference<?> ref2 = Mockito.mock(ServiceReference.class);
+        UsageCount uc2 = sr.obtainUsageCount(b, ref2, null, true);
+        assertEquals(1, inUseMap.size());
+        assertEquals(2, inUseMap.values().iterator().next().length);
+        List<UsageCount> ucl = Arrays.asList(inUseMap.get(b));
+        assertTrue(ucl.contains(uc));
+        assertTrue(ucl.contains(uc2));
+    }
+
+    public void testObtainUsageCountPrototypeUnknownLookup() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        @SuppressWarnings("unchecked")
+        ConcurrentMap<Bundle, UsageCount[]> inUseMap = (ConcurrentMap<Bundle, UsageCount[]>) getPrivateField(sr, "m_inUseMap");
+
+        Bundle b = Mockito.mock(Bundle.class);
+        ServiceReference<?> ref = Mockito.mock(ServiceReference.class);
+
+        UsageCount uc = new UsageCount(ref, true);
+        ServiceHolder sh = new ServiceHolder();
+        String svc = "foobar";
+        sh.m_service = svc;
+        uc.m_svcHolderRef.set(sh);
+        inUseMap.put(b, new UsageCount[] {uc});
+
+        assertNull(sr.obtainUsageCount(b, Mockito.mock(ServiceReference.class), null, null));
+
+        UsageCount uc2 = sr.obtainUsageCount(b, ref, svc, null);
+        assertSame(uc, uc2);
+    }
+
+    public void testObtainUsageCountPrototypeUnknownLookup2() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        @SuppressWarnings("unchecked")
+        ConcurrentMap<Bundle, UsageCount[]> inUseMap = (ConcurrentMap<Bundle, UsageCount[]>) getPrivateField(sr, "m_inUseMap");
+
+        Bundle b = Mockito.mock(Bundle.class);
+        ServiceReference<?> ref = Mockito.mock(ServiceReference.class);
+
+        UsageCount uc = new UsageCount(ref, false);
+        inUseMap.put(b, new UsageCount[] {uc});
+
+        assertNull(sr.obtainUsageCount(b, Mockito.mock(ServiceReference.class), null, null));
+
+        UsageCount uc2 = sr.obtainUsageCount(b, ref, null, null);
+        assertSame(uc, uc2);
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testObtainUsageCountRetry1() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        final Bundle b = Mockito.mock(Bundle.class);
+
+        final ConcurrentMap<Bundle, UsageCount[]> orgInUseMap =
+            (ConcurrentMap<Bundle, UsageCount[]>) getPrivateField(sr, "m_inUseMap");
+
+        ConcurrentMap<Bundle, UsageCount[]> inUseMap =
+            Mockito.mock(ConcurrentMap.class, AdditionalAnswers.delegatesTo(orgInUseMap));
+        Mockito.doAnswer(new Answer<UsageCount[]>()
+            {
+                @Override
+                public UsageCount[] answer(InvocationOnMock invocation) throws Throwable
+                {
+                    // This mimicks another thread putting another UsageCount in concurrently
+                    // The putIfAbsent() will fail and it has to retry
+                    UsageCount uc = new UsageCount(Mockito.mock(ServiceReference.class), false);
+                    UsageCount[] uca = new UsageCount[] {uc};
+                    orgInUseMap.put(b, uca);
+                    return uca;
+                }
+            }).when(inUseMap).putIfAbsent(Mockito.any(Bundle.class), Mockito.any(UsageCount[].class));
+        setPrivateField(sr, "m_inUseMap", inUseMap);
+
+        ServiceReference<?> ref = Mockito.mock(ServiceReference.class);
+
+        assertEquals(0, orgInUseMap.size());
+        UsageCount uc = sr.obtainUsageCount(b, ref, null, false);
+        assertEquals(1, orgInUseMap.size());
+        assertEquals(2, orgInUseMap.get(b).length);
+        assertSame(ref, uc.m_ref);
+        assertFalse(uc.m_prototype);
+        List<UsageCount> l = new ArrayList<UsageCount>(Arrays.asList(orgInUseMap.get(b)));
+        l.remove(uc);
+        assertEquals("There should be one UsageCount left", 1, l.size());
+        assertNotSame(ref, l.get(0).m_ref);
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testObtainUsageCountRetry2() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        final Bundle b = Mockito.mock(Bundle.class);
+
+        final ConcurrentMap<Bundle, UsageCount[]> orgInUseMap =
+            (ConcurrentMap<Bundle, UsageCount[]>) getPrivateField(sr, "m_inUseMap");
+        orgInUseMap.put(b, new UsageCount[] {new UsageCount(Mockito.mock(ServiceReference.class), false)});
+
+        ConcurrentMap<Bundle, UsageCount[]> inUseMap =
+            Mockito.mock(ConcurrentMap.class, AdditionalAnswers.delegatesTo(orgInUseMap));
+        Mockito.doAnswer(new Answer<Boolean>()
+            {
+                @Override
+                public Boolean answer(InvocationOnMock invocation) throws Throwable
+                {
+                    orgInUseMap.remove(b);
+                    return false;
+                }
+            }).when(inUseMap).replace(Mockito.any(Bundle.class),
+                    Mockito.any(UsageCount[].class), Mockito.any(UsageCount[].class));
+        setPrivateField(sr, "m_inUseMap", inUseMap);
+
+        ServiceReference<?> ref = Mockito.mock(ServiceReference.class);
+
+        assertEquals("Precondition", 1, inUseMap.size());
+        assertEquals("Precondition", 1, inUseMap.values().iterator().next().length);
+        assertNotSame("Precondition", ref, inUseMap.get(b)[0].m_ref);
+        sr.obtainUsageCount(b, ref, null, false);
+        assertEquals(1, inUseMap.size());
+        assertEquals(1, inUseMap.values().iterator().next().length);
+        assertSame("The old usage count should have been removed by the mock and this one should have been added",
+                ref, inUseMap.get(b)[0].m_ref);
+    }
+
+    public void testFlushUsageCount() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        @SuppressWarnings("unchecked")
+        ConcurrentMap<Bundle, UsageCount[]> inUseMap = (ConcurrentMap<Bundle, UsageCount[]>) getPrivateField(sr, "m_inUseMap");
+
+        Bundle b = Mockito.mock(Bundle.class);
+
+        ServiceReference<?> ref = Mockito.mock(ServiceReference.class);
+        UsageCount uc = new UsageCount(ref, false);
+        ServiceReference<?> ref2 = Mockito.mock(ServiceReference.class);
+        UsageCount uc2 = new UsageCount(ref2, true);
+
+        inUseMap.put(b, new UsageCount[] {uc, uc2});
+
+        assertEquals("Precondition", 1, inUseMap.size());
+        assertEquals("Precondition", 2, inUseMap.values().iterator().next().length);
+
+        sr.flushUsageCount(b, ref, uc);
+        assertEquals(1, inUseMap.size());
+        assertEquals(1, inUseMap.values().iterator().next().length);
+        assertSame(uc2, inUseMap.values().iterator().next()[0]);
+
+        sr.flushUsageCount(b, ref2, uc2);
+        assertEquals(0, inUseMap.size());
+    }
+
+    public void testFlushUsageCountNullRef() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        @SuppressWarnings("unchecked")
+        ConcurrentMap<Bundle, UsageCount[]> inUseMap = (ConcurrentMap<Bundle, UsageCount[]>) getPrivateField(sr, "m_inUseMap");
+
+        Bundle b = Mockito.mock(Bundle.class);
+        Bundle b2 = Mockito.mock(Bundle.class);
+
+        ServiceReference<?> ref = Mockito.mock(ServiceReference.class);
+        UsageCount uc = new UsageCount(ref, false);
+        ServiceReference<?> ref2 = Mockito.mock(ServiceReference.class);
+        UsageCount uc2 = new UsageCount(ref2, true);
+        ServiceReference<?> ref3 = Mockito.mock(ServiceReference.class);
+        UsageCount uc3 = new UsageCount(ref3, true);
+
+        inUseMap.put(b, new UsageCount[] {uc2, uc});
+        inUseMap.put(b2, new UsageCount[] {uc3});
+
+        assertEquals("Precondition", 2, inUseMap.size());
+
+        sr.flushUsageCount(b, null, uc);
+        assertEquals(2, inUseMap.size());
+
+        sr.flushUsageCount(b, null, uc2);
+        assertEquals(1, inUseMap.size());
+    }
+
+    public void testFlushUsageCountAlienObject() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        @SuppressWarnings("unchecked")
+        ConcurrentMap<Bundle, UsageCount[]> inUseMap = (ConcurrentMap<Bundle, UsageCount[]>) getPrivateField(sr, "m_inUseMap");
+
+        Bundle b = Mockito.mock(Bundle.class);
+
+        ServiceReference<?> ref = Mockito.mock(ServiceReference.class);
+        UsageCount uc = new UsageCount(ref, false);
+
+        inUseMap.put(b, new UsageCount[] {uc});
+        assertEquals("Precondition", 1, inUseMap.size());
+        assertEquals("Precondition", 1, inUseMap.values().iterator().next().length);
+
+        UsageCount uc2 = new UsageCount(Mockito.mock(ServiceReference.class), false);
+        sr.flushUsageCount(b, ref, uc2);
+        assertEquals("Should be no changes", 1, inUseMap.size());
+        assertEquals("Should be no changes", 1, inUseMap.values().iterator().next().length);
+    }
+
+    public void testFlushUsageCountNull() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        @SuppressWarnings("unchecked")
+        ConcurrentMap<Bundle, UsageCount[]> inUseMap = (ConcurrentMap<Bundle, UsageCount[]>) getPrivateField(sr, "m_inUseMap");
+
+        Bundle b = Mockito.mock(Bundle.class);
+        Bundle b2 = Mockito.mock(Bundle.class);
+
+        ServiceReference<?> ref = Mockito.mock(ServiceReference.class);
+        UsageCount uc = new UsageCount(ref, false);
+        ServiceReference<?> ref2 = Mockito.mock(ServiceReference.class);
+        UsageCount uc2 = new UsageCount(ref2, true);
+        ServiceReference<?> ref3 = Mockito.mock(ServiceReference.class);
+        UsageCount uc3 = new UsageCount(ref3, true);
+
+        inUseMap.put(b, new UsageCount[] {uc2, uc});
+        inUseMap.put(b2, new UsageCount[] {uc3});
+
+        assertEquals("Precondition", 2, inUseMap.size());
+
+        sr.flushUsageCount(b, ref, null);
+        assertEquals(2, inUseMap.size());
+
+        sr.flushUsageCount(b, ref2, null);
+        assertEquals(1, inUseMap.size());
+
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testFlushUsageCountRetry() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        final Bundle b = Mockito.mock(Bundle.class);
+        final ServiceReference<?> ref = Mockito.mock(ServiceReference.class);
+        final UsageCount uc = new UsageCount(ref, false);
+        final ServiceReference<?> ref2 = Mockito.mock(ServiceReference.class);
+        final UsageCount uc2 = new UsageCount(ref2, false);
+
+        final ConcurrentMap<Bundle, UsageCount[]> orgInUseMap =
+            (ConcurrentMap<Bundle, UsageCount[]>) getPrivateField(sr, "m_inUseMap");
+
+        final ConcurrentMap<Bundle, UsageCount[]> inUseMap =
+            Mockito.mock(ConcurrentMap.class, AdditionalAnswers.delegatesTo(orgInUseMap));
+        Mockito.doAnswer(new Answer<Boolean>()
+            {
+                @Override
+                public Boolean answer(InvocationOnMock invocation) throws Throwable
+                {
+                    inUseMap.put(b, new UsageCount[] {uc});
+                    return false;
+                }
+            }).when(inUseMap).replace(Mockito.isA(Bundle.class),
+                    Mockito.isA(UsageCount[].class), Mockito.isA(UsageCount[].class));
+        setPrivateField(sr, "m_inUseMap", inUseMap);
+
+        inUseMap.put(b, new UsageCount[] {uc, uc2});
+
+        sr.flushUsageCount(b, null, uc);
+
+        assertNull("A 'concurrent' process has removed uc2 as well, "
+                + "so the entry for 'b' should have been removed",
+                inUseMap.get(b));
+    }
+
+    public void testFlushUsageCountRetry2() throws Exception
+    {
+        ServiceRegistry sr = new ServiceRegistry(null, null);
+
+        final Bundle b = Mockito.mock(Bundle.class);
+        final ServiceReference<?> ref = Mockito.mock(ServiceReference.class);
+        final UsageCount uc = new UsageCount(ref, false);
+        final ServiceReference<?> ref2 = Mockito.mock(ServiceReference.class);
+        final UsageCount uc2 = new UsageCount(ref2, false);
+
+        final ConcurrentMap<Bundle, UsageCount[]> orgInUseMap =
+            (ConcurrentMap<Bundle, UsageCount[]>) getPrivateField(sr, "m_inUseMap");
+
+        final ConcurrentMap<Bundle, UsageCount[]> inUseMap =
+            Mockito.mock(ConcurrentMap.class, AdditionalAnswers.delegatesTo(orgInUseMap));
+        Mockito.doAnswer(new Answer<Boolean>()
+            {
+                @Override
+                public Boolean answer(InvocationOnMock invocation) throws Throwable
+                {
+                    inUseMap.put(b, new UsageCount[] {uc, uc2});
+                    return false;
+                }
+            }).when(inUseMap).remove(Mockito.isA(Bundle.class), Mockito.isA(UsageCount[].class));
+        setPrivateField(sr, "m_inUseMap", inUseMap);
+
+        inUseMap.put(b, new UsageCount[] {uc});
+
+        sr.flushUsageCount(b, null, uc);
+
+        assertEquals(1, inUseMap.get(b).length);
+        assertSame(uc2, inUseMap.get(b)[0]);
+    }
+
+    private Object getPrivateField(Object obj, String fieldName) throws NoSuchFieldException,
+            IllegalAccessException
+    {
+        Field f = ServiceRegistry.class.getDeclaredField(fieldName);
+        f.setAccessible(true);
+        return f.get(obj);
+    }
+
+    private void setPrivateField(ServiceRegistry obj, String fieldName, Object val) throws SecurityException,
+            NoSuchFieldException, IllegalArgumentException, IllegalAccessException
+    {
+        Field f = ServiceRegistry.class.getDeclaredField(fieldName);
+        f.setAccessible(true);
+        f.set(obj, val);
+    }
 }