sololis

Man, I wish I wasn't so tired as I'd like to explain this. This is positively the funniest thing I've come across all week. I'm starting to study the Solaris heap, not only because I want to learn it but because it's starting to look like I have a fairly large vulnerability in Solaris that of course deals with the heap, this is what I came across.

/* i may have missed a line here or there but atm this is what i think happens
 * (aka i havent actually gotten a sun box running well enough yet to test this)
 * WAY TO GO SUN!
 * -jf
 */

src/lib/libc/inc/mtlib.h:

[...]
#if defined(THREAD_DEBUG)
extern  void    assert_no_libc_locks_held(void);

#else
#define assert_no_libc_locks_held()
#endif

src/lib/libc/port/threads/sync.c:

[...]
#pragma weak _private_mutex_lock = __mutex_lock
[...]
int

__mutex_lock(mutex_t *mp)
{
        ASSERT(!curthread->ul_critical || curthread->ul_bindflags);

        return (mutex_lock_impl(mp, NULL));
}
[...]

static int
mutex_lock_impl(mutex_t *mp, timespec_t *tsp)
{

        ulwp_t *self = curthread;
        uberdata_t *udp = self->ul_uberdata;

        uberflags_t *gflags;
        int mtype;

        /*
         * Optimize the case of USYNC_THREAD, including
         * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases,
         * no error detection, no lock statistics,
         * and the process has only a single thread.
         * (Most likely a traditional single-threaded application.)
         */
        if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) |

            udp->uberflags.uf_all) == 0) {
                /*
                 * Only one thread exists so we don't need an atomic operation.
                 */

                if (mp->mutex_lockw == 0) {
                        mp->mutex_lockw = LOCKSET;

                        mp->mutex_owner = (uintptr_t)self;
                        DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);

                        return (0);
                }
                if (mtype && MUTEX_OWNER(mp) == self)

                        return (mutex_recursion(mp, mtype, MUTEX_LOCK));

                /*
                 * We have reached a deadlock, probably because the
                 * process is executing non-async-signal-safe code in
                 * a signal handler and is attempting to acquire a lock
                 * that it already owns.  This is not surprising, given
                 * bad programming practices over the years that has
                 * resulted in applications calling printf() and such
                 * in their signal handlers.  Unless the user has told
                 * us that the signal handlers are safe by setting:
                 *      export _THREAD_ASYNC_SAFE=1
                 * we return EDEADLK rather than actually deadlocking.
                 */
                if (tsp == NULL &&
                    MUTEX_OWNER(mp) == self && !self->ul_async_safe) {

                        DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
                        return (EDEADLK);

[...]

src/lib/libc/port/gen/malloc.c:

[...]
void *
malloc(size_t size)
{
        void *ret;

        if (!primary_link_map) {
                errno = ENOTSUP;

                return (NULL);
        }
        assert_no_libc_locks_held();

        (void) _private_mutex_lock(&libc_malloc_lock);
        ret = _malloc_unlocked(size);

        (void) _private_mutex_unlock(&libc_malloc_lock);
        return (ret);

}