Re: Avoiding linking with /home/packaging/X/libxx.so

To: debian-mentors <debian-mentors@lists.debian.org>
Subject: Re: Avoiding linking with /home/packaging/X/libxx.so
From: lumin <cdluminate@gmail.com>
Date: Sun, 14 Jun 2015 02:34:40 +0000
Message-id: <[🔎] 1434249280.7036.51.camel@gmail.com>
In-reply-to: <557CDFC0.2060108@163.com>
References: <[🔎] 20150613114845.GA4748@jwilk.net> <557CDFC0.2060108@163.com>

Hi mentors,

> 
> That's not the unusual way to link to a shared library. You should use
> something like:
> 
> 	$(CC) -g -o hello hello.o -L../lib/ -lsharedlib
> 
> (Although it would probably only work if the library had a SONAME.)

I did an experiment. Had the rpath issue fixed, the build result seems
to be far much better! and no any ELF contains RPATH after modifying 
upstream Makefile (full quilt patch attached as [01-*.patch]).

I installed the experimentally built packages, and they works well.

$ readelf -d libcaffe.so.0 
        0x000000000000000e (SONAME)  Library soname: [libcaffe.so.0]
$ readelf -d caffe.bin 
        0x0000000000000001 (NEEDED)  Shared library: [libcaffe.so.0]
$ ldd caffe.bin   | grep caffe
        libcaffe.so.0 => /usr/lib/libcaffe.so.0 (0x00007f8e1d222000)

1. I added SONAME for shlib, changing upstream makefile.
        it is implemented by a hack:
        
 44 +   #$(Q)$(CXX) -shared -o $@ $(OBJS) $(LINKFLAGS) $(LDFLAGS)
$(DYNAMIC_FLAGS)
 45 +   echo $(CXX) -shared -o $@ $(OBJS) $(LINKFLAGS) $(LDFLAGS)
$(DYNAMIC_FLAGS) \
 46 +       -Wl,-soname,$(SONAME) > build_so.sh
 47 +   sed -i -e 's/-pie//g' build_so.sh
 48 +   sh build_so.sh
 49 +   sh -c "cd $(LIB_BUILD_DIR); ln -s $(SONAME) libcaffe.so"
        
        it looks weird, but I didn't find the source of '-pie' option,
        in order to remove -pie option, so used a dirty hack.
        if I don't remove -pie, g++ would complain: cannot find
        reference to
        main , such thing. then failed to build.
        this dirty hack works very well.

2. In upstream Makefile, I replaced -rpath with -L$(..). globally,
   e.g.

 80  $(TOOL_BINS): %.bin : %.o | $(DYNAMIC_NAME)
 81     @ echo CXX/LD -o $@
 82     $(Q)$(CXX) $< -o $@ $(LINKFLAGS) -l$(PROJECT) $(LDFLAGS) \
 83 -       -Wl,-rpath,$(ORIGIN)/../lib
 84 +       -L$(ORIGIN)/$(LIB_BUILD_DIR)/

> * Andrey Rahmatullin <wrar@debian.org>, 2015-06-12, 14:31:
> >SONAMEs, not paths, are added to DT_NEEDED.
> 
> Technically, you can put (both relative and absolute) paths in
> DT_NEEDED, not only SONAMEs. The dynamic linker is happy to use such
> DT_NEEDED entries. It's a terrible idea, though. :-)

Well, now I added SONAME, and the test build looks very good.
Now I can continue with my ITP.
FYI: I also attached (experimental)rules and (experimental)control,
if someone is interested.

Thank you all for answering my questions :-)

-- 
Regards,
  C.D.Luminate

Fix rpath issue in upstream Makefile
--- a/Makefile
+++ b/Makefile
@@ -1,3 +1,7 @@
+# start debian
+SONAME := libcaffe.so.0
+Q :=
+# end debian
 PROJECT := caffe
 
 CONFIG_FILE := Makefile.config
@@ -31,7 +35,7 @@
 # The target shared library name
 LIB_BUILD_DIR := $(BUILD_DIR)/lib
 STATIC_NAME := $(LIB_BUILD_DIR)/lib$(PROJECT).a
-DYNAMIC_NAME := $(LIB_BUILD_DIR)/lib$(PROJECT).so
+DYNAMIC_NAME := $(LIB_BUILD_DIR)/$(SONAME)
 
 ##############################
 # Get all source files
@@ -255,7 +259,7 @@
 	# boost::thread is called boost_thread-mt to mark multithreading on OS X
 	LIBRARIES += boost_thread-mt
 	# we need to explicitly ask for the rpath to be obeyed
-	DYNAMIC_FLAGS := -install_name @rpath/libcaffe.so
+	DYNAMIC_FLAGS := -Wl,-soname,$(SONAME)
 	ORIGIN := @loader_path
 else
 	ORIGIN := \$$ORIGIN
@@ -443,7 +447,7 @@
 	@ echo CXX/LD -o $@ $<
 	$(Q)$(CXX) -shared -o $@ $(PY$(PROJECT)_SRC) \
 		-o $@ $(LINKFLAGS) -l$(PROJECT) $(PYTHON_LDFLAGS) \
-		-Wl,-rpath,$(ORIGIN)/../../build/lib
+		-L$(ORIGIN)/$(LIB_BUILD_DIR)/
 
 mat$(PROJECT): mat
 
@@ -506,7 +510,12 @@
 
 $(DYNAMIC_NAME): $(OBJS) | $(LIB_BUILD_DIR)
 	@ echo LD -o $@
-	$(Q)$(CXX) -shared -o $@ $(OBJS) $(LINKFLAGS) $(LDFLAGS) $(DYNAMIC_FLAGS)
+	#$(Q)$(CXX) -shared -o $@ $(OBJS) $(LINKFLAGS) $(LDFLAGS) $(DYNAMIC_FLAGS)
+	echo $(CXX) -shared -o $@ $(OBJS) $(LINKFLAGS) $(LDFLAGS) $(DYNAMIC_FLAGS) \
+		-Wl,-soname,$(SONAME) > build_so.sh
+	sed -i -e 's/-pie//g' build_so.sh
+	sh build_so.sh
+	sh -c "cd $(LIB_BUILD_DIR); ln -s $(SONAME) libcaffe.so"
 
 $(STATIC_NAME): $(OBJS) | $(LIB_BUILD_DIR)
 	@ echo AR -o $@
@@ -537,19 +546,22 @@
 		| $(DYNAMIC_NAME) $(TEST_BIN_DIR)
 	@ echo CXX/LD -o $@ $<
 	$(Q)$(CXX) $(TEST_MAIN_SRC) $(TEST_OBJS) $(GTEST_OBJ) \
-		-o $@ $(LINKFLAGS) $(LDFLAGS) -l$(PROJECT) -Wl,-rpath,$(ORIGIN)/../lib
+		-o $@ $(LINKFLAGS) $(LDFLAGS) -l$(PROJECT) \
+		-L$(ORIGIN)/$(LIB_BUILD_DIR)/
 
 $(TEST_CU_BINS): $(TEST_BIN_DIR)/%.testbin: $(TEST_CU_BUILD_DIR)/%.o \
 	$(GTEST_OBJ) | $(DYNAMIC_NAME) $(TEST_BIN_DIR)
 	@ echo LD $<
 	$(Q)$(CXX) $(TEST_MAIN_SRC) $< $(GTEST_OBJ) \
-		-o $@ $(LINKFLAGS) $(LDFLAGS) -l$(PROJECT) -Wl,-rpath,$(ORIGIN)/../lib
+		-o $@ $(LINKFLAGS) $(LDFLAGS) -l$(PROJECT) \
+		-L$(ORIGIN)/$(LIB_BUILD_DIR)/
 
 $(TEST_CXX_BINS): $(TEST_BIN_DIR)/%.testbin: $(TEST_CXX_BUILD_DIR)/%.o \
 	$(GTEST_OBJ) | $(DYNAMIC_NAME) $(TEST_BIN_DIR)
 	@ echo LD $<
 	$(Q)$(CXX) $(TEST_MAIN_SRC) $< $(GTEST_OBJ) \
-		-o $@ $(LINKFLAGS) $(LDFLAGS) -l$(PROJECT) -Wl,-rpath,$(ORIGIN)/../lib
+		-o $@ $(LINKFLAGS) $(LDFLAGS) -l$(PROJECT) \
+		-L$(ORIGIN)/$(LIB_BUILD_DIR)/
 
 # Target for extension-less symlinks to tool binaries with extension '*.bin'.
 $(TOOL_BUILD_DIR)/%: $(TOOL_BUILD_DIR)/%.bin | $(TOOL_BUILD_DIR)
@@ -559,12 +571,12 @@
 $(TOOL_BINS): %.bin : %.o | $(DYNAMIC_NAME)
 	@ echo CXX/LD -o $@
 	$(Q)$(CXX) $< -o $@ $(LINKFLAGS) -l$(PROJECT) $(LDFLAGS) \
-		-Wl,-rpath,$(ORIGIN)/../lib
+		-L$(ORIGIN)/$(LIB_BUILD_DIR)/
 
 $(EXAMPLE_BINS): %.bin : %.o | $(DYNAMIC_NAME)
 	@ echo CXX/LD -o $@
 	$(Q)$(CXX) $< -o $@ $(LINKFLAGS) -l$(PROJECT) $(LDFLAGS) \
-		-Wl,-rpath,$(ORIGIN)/../../lib
+		-L$(ORIGIN)/$(LIB_BUILD_DIR)/
 
 proto: $(PROTO_GEN_CC) $(PROTO_GEN_HEADER)
 
@@ -588,6 +600,7 @@
 	@- $(RM) -rf $(DISTRIBUTE_DIR)
 	@- $(RM) $(PY$(PROJECT)_SO)
 	@- $(RM) $(MAT$(PROJECT)_SO)
+	-rm build_so.sh
 
 supercleanfiles:
 	$(eval SUPERCLEAN_FILES := $(strip \
@@ -618,6 +631,20 @@
 	# add include
 	cp -r include $(DISTRIBUTE_DIR)/
 	mkdir -p $(DISTRIBUTE_DIR)/include/caffe/proto
+	cp $(PROTO_GEN_HEADER_SRCS) $(DISTRIBUTE_DIR)/include/caffe/proto
+	# add tool and example binaries
+	cp $(TOOL_BINS) $(DISTRIBUTE_DIR)/bin
+	cp $(EXAMPLE_BINS) $(DISTRIBUTE_DIR)/bin
+	# add libraries
+	cp $(STATIC_NAME) $(DISTRIBUTE_DIR)/lib
+	cp $(DYNAMIC_NAME) $(DISTRIBUTE_DIR)/lib
+	# add python - it's not the standard way, indeed...
+	cp -r python $(DISTRIBUTE_DIR)/python
+
+__distribute:
+	# add include
+	cp -r include $(DISTRIBUTE_DIR)/
+	mkdir -p $(DISTRIBUTE_DIR)/include/caffe/proto
 	cp $(PROTO_GEN_HEADER_SRCS) $(DISTRIBUTE_DIR)/include/caffe/proto
 	# add tool and example binaries
 	cp $(TOOL_BINS) $(DISTRIBUTE_DIR)/bin

Source: caffe
Section: contrib/science
Priority: extra
Maintainer: Zhou Mo <cdluminate@gmail.com>
Build-Depends: debhelper (>=9)
 , libopencv-dev (>= 2.4)
 , libboost-all-dev (>= 1.55) | libboost1.55-all-dev (>= 1.55)
 , libatlas-dev | libopenblas-dev
 , libprotobuf-dev
 , libleveldb-dev
 , libsnappy-dev
 , libhdf5-serial-dev
 , libgflags-dev
 , libgoogle-glog-dev
 , liblmdb-dev
 , protobuf-compiler
 , nvidia-cuda-toolkit
Standards-Version: 3.9.6
Homepage: http://caffe.berkeleyvision.org
Vcs-Git: git://github.com/BVLC/caffe.git
Vcs-Browser: https://github.com/BVLC/caffe

Package: caffe-cpu-atlas
Section: science
Architecture: any
Multi-Arch: foreign
Depends: ${misc:Depends}, ${shlibs:Depends}
Suggests: caffe-doc, caffe-data
Conflicts: caffe-cpu-oblas
 , caffe-gpu-atlas
 , caffe-gpu-oblas
Description: auto-generated package by debmake: bin #0
 This package contains the compiled binary executable.: #0
 .
 This Debian binary package was auto-generated by the
 debmake(1) command provided by the debmake package.

Package: caffe-cpu-openblas
Section: science
Architecture: any
Multi-Arch: foreign
Depends: ${misc:Depends}, ${shlibs:Depends}
Suggests: caffe-doc, caffe-data
Conflicts: caffe-cpu-atlas
 , caffe-gpu-atlas
 , caffe-gpu-oblas
Description: auto-generated package by debmake: bin #1
 This package contains the compiled binary executable.: #1

#Package: caffe-gpu-atlas
#Section: contrib/science
#Architecture: any
#Multi-Arch: foreign
#Depends: ${misc:Depends}, ${shlibs:Depends}
#Suggests: caffe-doc, caffe-data
#Conflicts: caffe-cpu-atlas
# , caffe-cpu-oblas
# , caffe-gpu-oblas
#Description: auto-generated package by debmake: bin #2
# This package contains the compiled binary executable.: #2
#
#Package: caffe-gpu-openblas
#Section: contrib/science
#Architecture: any
#Multi-Arch: foreign
#Depends: ${misc:Depends}, ${shlibs:Depends}
#Suggests: caffe-doc, caffe-data
#Conflicts: caffe-cpu-atlas
# , caffe-cpu-oblas
# , caffe-gpu-atlas
#Description: auto-generated package by debmake: bin #3
# This package contains the compiled binary executable.: #3

Package: caffe-doc
Section: doc
Architecture: all
Multi-Arch: foreign
Depends: ${misc:Depends}
Description: auto-generated package by debmake: doc #4
 This package contains the documentation files.: #4

Package: caffe-data
Architecture: all
Multi-Arch: foreign
Depends: ${misc:Depends}
Description: auto-generated package by debmake: data #5
 This package contains the architecture independent data.: #5

#!/usr/bin/make -f
export DH_VERBOSE = 1
export DEB_BUILD_MAINT_OPTIONS = hardening=+all
export DEB_CFLAGS_MAINT_APPEND  = -Wall -pedantic
export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed

RULES := debian/rules
CONF_DIR := $(CURDIR)/debian/config
TOP := $(CURDIR)
#TEMP := $(shell mktemp -u -d -p $(TOP)/../)/
TEMP := $(TOP)/../__temp_caffe__/

.PHONY: config CA_conf CO_conf GA_conf GO_conf rmconf
config: CA_conf
CA_conf:
	cp $(CONF_DIR)/cpu-atlas.conf $(TOP)/Makefile.config
CO_conf:
	cp $(CONF_DIR)/cpu-oblas.conf $(TOP)/Makefile.config
GA_conf:
	cp $(CONF_DIR)/gpu-atlas.conf $(TOP)/Makefile.config
GO_conf:
	cp $(CONF_DIR)/gpu-oblas.conf $(TOP)/Makefile.config
rmconf:
	-rm Makefile.config

%:
	dh $@  

override_dh_auto_clean: config
	dh_auto_clean
	-rm Makefile.config
	-rm -rf d_*

override_dh_auto_build:
	mkdir $(TEMP)
	# Build 4 versions of caffe
	$(RULES) build_CA
	dh clean
	$(RULES) build_CO
	dh clean
	#$(RULES) build_GA	
	#dh clean
	#$(RULES) build_GO	
	#dh clean
	# Copy distribute files of 4 versions back
	cp -r $(TEMP)/* .
	rm -rf $(TEMP)

override_dh_auto_test:
	# FIXME: skip test ?
	#$(MAKE) test
	#$(MAKE) runtest

override_dh_auto_install:
	# upstream Makefile contains NO install target, skipping.
	true

build_CA:
	# build cpu-atlas
	$(RULES) CA_conf
	$(MAKE) -j4 all
	$(MAKE) __distribute
	$(RULES) rmconf
	## save distribute files
	cp -r d_cpu_atlas $(TEMP)/
build_CO:
	# build cpu-oblas
	$(RULES) CO_conf
	$(MAKE) -j4 all
	$(MAKE) __distribute
	$(RULES) rmconf
	## save distribute files
	cp -r d_cpu_oblas $(TEMP)/
build_GA:
	# build gpu-atlas
	$(RULES) GA_conf
	$(MAKE) -j4 all
	$(MAKE) __distribute
	$(RULES) rmconf
	## save distribute files
	cp -r d_gpu_atlas $(TEMP)/
build_GO:
	# build gpu-oblas
	$(RULES) GO_conf
	$(MAKE) -j4 all
	$(MAKE) __distribute
	$(RULES) rmconf
	## save distribute files
	cp -r d_gpu_oblas $(TEMP)/

Reply to:

References:
- Re: Avoiding linking with /home/packaging/X/libxx.so
  - From: Jakub Wilk <jwilk@debian.org>

Prev by Date: Bug#788653: RFS: ktap/0.4-1 [ITP] -- lightweight script-based dynamic tracing tool for Linux
Next by Date: Bug#775532: citeproc/data/schema/csl.rng license Re: Bug#775532: RFS: citeproc-py/0.3.0-1 [ITP] -- Python library for CSL based bibliography processing
Previous by thread: Re: Avoiding linking with /home/packaging/X/libxx.so
Next by thread: Fwd: Introducing myself - Joël Krähemann
Index(es):
- Date
- Thread